{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LeNet on Cifar\n",
    "\n",
    "This is LeNet (6c-16c-120-84) on Cifar10. Adam algorithm (lr=0.001) with 100 epoches.\n",
    "\n",
    "\n",
    "#### LeNet\n",
    "\n",
    "    Total params: 44,426\n",
    "    Trainable params: 44,426\n",
    "    Non-trainable params: 0\n",
    "\n",
    "\n",
    "####  LeNet with 10 intrinsic dim\n",
    "\n",
    "    Total params: 682,076\n",
    "    Trainable params: 10\n",
    "    Non-trainable params: 682,066\n",
    "    \n",
    "#### LeNet with 15000 intrinsic dim    \n",
    "    Total params: 930,167,006\n",
    "    Trainable params: 15,000\n",
    "    Non-trainable params: 930,152,006"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os, sys\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import *\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_num(lines0):\n",
    "\n",
    "    valid_loss_str     = lines0[-5]\n",
    "    valid_accuracy_str = lines0[-6]\n",
    "    train_loss_str     = lines0[-8]\n",
    "    train_accuracy_str = lines0[-9]\n",
    "    run_time_str       = lines0[-10]\n",
    "\n",
    "    valid_loss     = float(valid_loss_str.split( )[-1])\n",
    "    valid_accuracy = float(valid_accuracy_str.split( )[-1])\n",
    "    train_loss     = float(train_loss_str.split( )[-1])\n",
    "    train_accuracy = float(train_accuracy_str.split( )[-1])\n",
    "    run_time       = float(run_time_str.split( )[-1])\n",
    "    \n",
    "    return valid_loss, valid_accuracy, train_loss, train_accuracy, run_time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 250, 500, 750, 1000, 1250, 1500, 1750, 1900, 1950, 2000, 2050, 2100, 2250, 2500, 5000, 10000, 15000, 20000, 25000, 30000, 35000, 40000, 45000, 50000]\n",
      "0 dim:\n",
      "(4.97572, 0.5796, 0.112174, 0.96086, 184.707)\n",
      "\n",
      "250 dim:\n",
      "(1.90655, 0.318, 1.90521, 0.31826, 212.042)\n",
      "\n",
      "500 dim:\n",
      "(1.67792, 0.3904, 1.67412, 0.39842, 217.218)\n",
      "\n",
      "750 dim:\n",
      "(1.55566, 0.4478, 1.54047, 0.45106, 217.959)\n",
      "\n",
      "1000 dim:\n",
      "(1.51612, 0.4622, 1.48594, 0.47296, 208.912)\n",
      "\n",
      "1250 dim:\n",
      "(1.45248, 0.4857, 1.42595, 0.49468, 209.329)\n",
      "\n",
      "1500 dim:\n",
      "(1.44311, 0.4804, 1.40013, 0.49982, 250.536)\n",
      "\n",
      "1750 dim:\n",
      "(1.37514, 0.5181, 1.31392, 0.53806, 211.105)\n",
      "\n",
      "1900 dim:\n",
      "(1.39473, 0.5025, 1.33424, 0.52604, 223.799)\n",
      "\n",
      "1950 dim:\n",
      "(1.37309, 0.5108, 1.32478, 0.5303, 222.919)\n",
      "\n",
      "2000 dim:\n",
      "(1.35499, 0.514, 1.29992, 0.53972, 231.783)\n",
      "\n",
      "2050 dim:\n",
      "(1.3594, 0.5185, 1.30346, 0.54162, 231.263)\n",
      "\n",
      "2100 dim:\n",
      "(1.36086, 0.5206, 1.29395, 0.54252, 229.522)\n",
      "\n",
      "2250 dim:\n",
      "(1.32558, 0.528, 1.27493, 0.5487, 222.882)\n",
      "\n",
      "2500 dim:\n",
      "(1.32359, 0.5344, 1.26351, 0.55712, 252.905)\n",
      "\n",
      "5000 dim:\n",
      "(1.23719, 0.5661, 1.10628, 0.61232, 254.977)\n",
      "\n",
      "10000 dim:\n",
      "(1.15696, 0.6081, 0.894347, 0.68416, 325.688)\n",
      "\n",
      "15000 dim:\n",
      "(1.20768, 0.6072, 0.772271, 0.7278, 399.654)\n",
      "\n",
      "20000 dim:\n",
      "(1.29638, 0.6161, 0.668575, 0.7646, 473.679)\n",
      "\n",
      "25000 dim:\n",
      "(1.36316, 0.6163, 0.592406, 0.78838, 568.811)\n",
      "\n",
      "30000 dim:\n",
      "(1.58397, 0.595, 0.513422, 0.81548, 629.579)\n",
      "\n",
      "35000 dim:\n",
      "(1.84335, 0.609, 0.423732, 0.84598, 707.691)\n",
      "\n",
      "40000 dim:\n",
      "(2.1227, 0.5856, 0.385001, 0.8584, 773.699)\n",
      "\n",
      "45000 dim:\n",
      "(2.28262, 0.5864, 0.333573, 0.87906, 915.193)\n",
      "\n",
      "50000 dim:\n",
      "(2.58714, 0.5696, 0.331673, 0.87816, 986.543)\n",
      "\n",
      "Intrinsic dim is: 2250 for LeNet\n"
     ]
    }
   ],
   "source": [
    "results_dir = '../results/lenet_cifar_l2_0.001'\n",
    "\n",
    "# dim = [0,250,500,750,1000,1250,1500,1750,1900,1950,2000,2050,2100,2250,2500,3000,4000,5000,5250,5500,5750,6000,6250,6500,6750,7000,7250,7500,7750,8000,8250,8500,8750,9000,9250,9500,9750,10000,15000,20000,25000,30000,35000,40000,45000,50000]\n",
    "\n",
    "dim = [0,10,50,100,500,750,1000,1250,1500,1750,1900,1950,2000,2050,2100,2250,2500,5000,10000,15000]\n",
    "i = 0        \n",
    "\n",
    "# filename list of diary\n",
    "diary_names = []\n",
    "for subdir, dirs, files in os.walk(results_dir):\n",
    "    for file in files:\n",
    "        if file == 'diary':\n",
    "            fname = os.path.join(subdir, file)\n",
    "            diary_names.append(fname)\n",
    "\n",
    "dim_ = []            \n",
    "for f in diary_names: \n",
    "    dim_.append( int(f.split('_')[-2]) )         \n",
    "    \n",
    "dim = sorted(dim_) \n",
    "\n",
    "print dim\n",
    "            \n",
    "diary_names_ordered = []\n",
    "for d in dim:\n",
    "    for f in diary_names:\n",
    "        if d == int(f.split('_')[-2]):\n",
    "            # print \"%d is in\" % d + f\n",
    "            diary_names_ordered.append(f)        \n",
    "\n",
    "# intrinsic update method\n",
    "Rs, Acc = [], []\n",
    "i = 0\n",
    "for fname in diary_names_ordered:\n",
    "    with open(fname,'r') as ff:\n",
    "        lines0 = ff.readlines()\n",
    "        R = extract_num(lines0)\n",
    "        print \"%d dim:\\n\"%dim[i] + str(R) + \"\\n\"\n",
    "        i += 1\n",
    "\n",
    "        Rs.append(R)\n",
    "        Acc.append(R[1])\n",
    "                            \n",
    "Rs = np.array(Rs)\n",
    "Acc = np.array(Acc)\n",
    "\n",
    "# 2.2 construct acc_solved_all and dim_solved_all           \n",
    "\n",
    "for id_d in range(len(dim)):\n",
    "    d = dim[id_d]\n",
    "\n",
    "    r = Rs[id_d,1]\n",
    "    if d==0:\n",
    "        test_acc_bl = r        \n",
    "        # print \"Acc goal is: \" + str(test_acc_sl) + \" for network with depth \" + str(ll) + \" width \"+ str(w)\n",
    "    else:\n",
    "        test_acc = r\n",
    "        if test_acc>test_acc_bl*0.9:\n",
    "            acc_solved=test_acc\n",
    "            dim_solved=d\n",
    "            print \"Intrinsic dim is: \" + str(d) + \" for LeNet\"\n",
    "            # print \"\\n\"\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0.5796],[250, 0.318],[500, 0.3904],[750, 0.4478],[1000, 0.4622],[1250, 0.4857],[1500, 0.4804],[1750, 0.5181],[1900, 0.5025],[1950, 0.5108],[2000, 0.514],[2050, 0.5185],[2100, 0.5206],[2250, 0.528],[2500, 0.5344],[5000, 0.5661],[10000, 0.6081],[15000, 0.6072],[20000, 0.6161],[25000, 0.6163],[30000, 0.595],[35000, 0.609],[40000, 0.5856],[45000, 0.5864],[50000, 0.5696]\n"
     ]
    }
   ],
   "source": [
    "print ','.join(['[%i, %s]' % (dim[n], Acc[n]) for n in xrange(len(Acc))])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Performance comparison with Baseline\n",
    "\n",
    "\"Baseline method\" indicates optimization in the parameter space.\n",
    "\n",
    "The proposed method first embeds parameters into the intrinisic space (via orthogonal matrix), and optimization is the new space.\n",
    "\n",
    "The dimension of intrinsic space indicates the degree of freedom in the weights of neural nets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAFFCAYAAAAdAsFPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XucVVX9//HXB0RB5X4XUGFCE5MUCO3rlziWCqSJphUg\n/RQL9BsIaPRNkqhQU/uaeUNNCrWQSxcVTJEkHUpJAZNQMJChiAEJAYHhItfP74+1ZziMM+ecgdlz\nhnPez8djP85Za6+991ozZ85n9l57r2XujoiISCp1sl0BERGp/RQsREQkLQULERFJS8FCRETSUrAQ\nEZG0FCxERCStWIOFmfU1s+VmttLMbqlg/c/MbHG0rDCzLUnrrjGz96LlmjjrKSIiqVlcz1mYWV1g\nBXARUAwsBAa6+7JKyt8InOPu15lZM2AR0ANw4E2gu7t/GEtlRUQkpTjPLHoCK919lbvvAaYD/VOU\nHwhMi973AV5y981RgHgJ6BtjXUVEJIU4g0U7YE1SujjK+xgzOwXoCLxc1W1FRCR+x2S7ApEBwO/c\nfX9VNjKzYcAwgAYNGnTv0KFDlQ984MAB6tTJr35+tTl/5GO71eaqWbFixUZ3b5muXJzBYi2Q/O3d\nPsqryABgeLltE+W2LSy/kbs/BjwG0KNHD1+0aFGVK1lYWEgikUhbLpeozfkjH9utNleNma3OpFyc\n4Xch0NnMOprZsYSAMKt8ITP7JNAU+GtS9hzgYjNramZNgYujPBERyYLYzizcfZ+ZjSB8ydcFJrv7\nUjObACxy99LAMQCY7km3Zbn7ZjO7jRBwACa4++a46ioiIqnF2mfh7i8AL5TLG18u/cNKtp0MTI6t\nciIikrH86gUSEZHDUlvuhhKRPHHgwAE2btzIli1b2L+/SjdAZqRx48a8++671b7f2ixdm+vXr0/7\n9u2pV6/eYR9DwUJEalRxcTFmxqmnnkq9evUws2rdf0lJCQ0bNqzWfdZ2qdrs7mzatIni4mI6dux4\n2MfQZSgRqVE7duygXbt2HHvssdUeKOTjzIzmzZvz0UcfHdF+FCxEpMbl20Nz2VYdQVm/MRERSUvB\nQkQkyamnnsrcuXNr7HhmxsqVKwG44YYbuO2222rs2FWhDm4RkVri0UcfzXYVKqUzCxERSUvBQkSk\nnIULF9KlSxeaNm3KkCFD+Oijj/jwww+59NJLadmyJU2bNuXSSy+luLi4bJsnnniCTp060bBhQzp2\n7MhTTz1Vtm7y5MmcccYZNG3alD59+rB6dcVj91177bWMGzcOCIMDtm/fnp/+9Ke0atWKtm3b8vjj\nj5eV3b17N2PGjOHkk0+moKCAG264gV27dsX0E1GwEBH5mKeeeoo5c+ZQVFTEihUruP322zlw4ABD\nhgxh9erV/Pvf/6ZBgwaMGDECCLcDjxw5ktmzZ1NSUsL8+fM5++yzAZg5cyY//vGPefrpp/nggw/o\n1asXAwcOzKge69evZ+vWraxdu5Zf/vKXDB8+nA8/DBOG3nLLLaxYsYLFixezePFi1q5dy4QJE+L5\ngaA+CxHJstGjR7N48eJq29/+/fupW7fuIXlnn3029913X8b7GDFiBKXz49x6663ceOON3H777Vx5\n5ZVlZW699VYuuOCCsnSdOnV45513OPnkk2nbti1t27YFQj/E2LFjOeOMMwD43ve+x49//GNWr17N\nKaeckrIe9erVY/z48RxzzDF88Ytf5MQTT2T58uWce+65PPbYYyxZsoRmzZpRUlLC9773PQYNGsSd\nd96ZcTurQmcWIiLlJE+kdsopp7Bu3Tp27tzJ9ddfzymnnEKjRo343Oc+VzZkyQknnMCMGTN49NFH\nadu2LZdccgn/+Mc/AFi9ejWjRo2iSZMmNGnShGbNmuHurF1b2fQ+BzVv3pxjjjn4P/3xxx/P9u3b\n+eCDD9i5cyfdu3enSZMmdOjQgb59+/LBBx9U/w8jojMLEcmqqvzHn4nqGO5jzZqDszr/+9//5qST\nTuKnP/0py5cv54033qBNmzYsXryYc845h9LZFfr06UOfPn3YtWsX48aNY+jQofzlL3+hQ4cO3Hrr\nrVx99dVHVKdkLVq0oEGDBixdupR27drVyBAnOrMQESln4sSJFBcXs3nzZu644w6+9rWvUVJSQoMG\nDWjSpAmbN2/mRz/6UVn5//znP8ycOZMdO3Zw3HHHceKJJ5Y9pX7DDTdw5513snTpUgC2bt3Kb3/7\n2yOqX506dRg6dCg33XQTGzZsAGDt2rXMmRPfHHEKFiIi5QwaNIiLL76YTp06UVBQwLhx4xg9ejS7\ndu2iRYsWnHfeefTt27es/IEDB7j33ns56aSTaNasGfPmzeORRx4B4IorruC73/0uAwYMoFGjRnzq\nU59i9uzZR1zHu+++m0984hOcd955tGvXjgsvvJDly5cf8X4rY0kT1B3VNAd35tTm/FEb2/3uu++W\ndfbGQaPOVqyyn7uZvenuPdIdQ2cWIiKSloKFiIikpWAhIiJpKViIiEhaChYiIpKWgoWIiKSlYCEi\nImkpWIiISFqxBgsz62tmy81spZndUkmZr5rZMjNbamZTk/L3m9niaJkVZz1FRI5E6dwT1e1f//oX\nZsa+ffuqfd9VFdtAgmZWF5gIXAQUAwvNbJa7L0sq0xkYC5zv7h+aWaukXexy97Pjqp+IiGQuzjOL\nnsBKd1/l7nuA6UD/cmWGAhPd/UMAd98QY31EROQwxRks2gFrktLFUV6y04DTzOw1M3vdzPomratv\nZoui/MtjrKeIyCHuvvtu2rVrR8OGDTn99NP505/+xO7duxk9ejQnnXQSJ510EqNHj2b37t0VbnvV\nVVcdkjdq1ChGjhwJhFFnv/GNb9C2bVvatWvHuHHj2L9/PxAmbhozZgwtWrSgU6dOPP/88/E3NkPZ\nns/iGKAzkADaA382s7PcfQtwiruvNbNOwMtm9ra7FyVvbGbDgGEArVu3prCwsMoV2L59+2FtdzRT\nm/NHbWx348aNKSkpiW3/+/fvP6L9v/feezz44IO88sortG3bltWrV7N//37Gjx/Pa6+9xquvvoqZ\nMWDAAL7//e/z/e9/n507d+LulJSUcOmll/KjH/2IdevW0bBhQ/bv38+MGTN46qmnKCkpYfDgwbRs\n2ZK33nqLnTt38pWvfIWWLVty3XXX8ctf/pJZs2bxl7/8heOPP56vf/3rQBgoMHkSpMNp80cffXRk\nnwV3j2UBPgvMSUqPBcaWK/MoMCQp/SfgMxXs6wngqlTH6969ux+OV1555bC2O5qpzfmjNrZ72bJl\nH8vr3bu3P/744+7uvmfPHu/du7f/+te/dnf3HTt2eO/evX369Onu7r5lyxbv3bu3//73v3d39w8+\n+MB79+7ts2bNcnf39957z3v37u2zZ892d/d///vfVarfe++95y1btvSXXnrJ9+zZU5bfqVMnf/75\n58vSL774op9yyinuHn7O7dq1K1t3/vnn+5NPPunu7n/84x+9U6dO7u6+fv16P/bYY33nzp1lZadO\nneqJRMLd3S+44AJ/5JFHytbNmTPHAd+7d2/KOm/bti1tuyr6ubu7A4s8g+/0OC9DLQQ6m1lHMzsW\nGACUv6vpWcJZBWbWgnBZapWZNTWz45LyzweWISISs0984hPcd999/PCHP6RVq1YMGDCAdevWsW7d\nukPmzC6dbrUigwYNYtq0aQBMnTqVQYMGAWGK1b1799K2bduyaVavv/76sgmM1q1b97EpXWuL2C5D\nufs+MxsBzAHqApPdfamZTSBEslnRuovNbBmwH/iOu28ys/8Cfm5mBwj9Knd50l1UIpJbki+P1KtX\n75D08ccff0i6cePGh6RbtGhxSLr8JenkL99MDRo0iEGDBrFt2zauv/56vvvd73LSSSexevVqzjzz\nTODgdKsV+cpXvsK3v/1tiouLeeaZZ/jrX/9aVpfjjjuOjRs3VnhZqW3bth+b0rW2iPU5C3d/wd1P\nc/cCd78jyhsfBQqis6Cb3b2Lu5/l7tOj/PlR+tPR6y/jrKeISKnly5fz8ssvs3v3burXr0+DBg2o\nU6cOAwcO5Pbbb+eDDz5g48aNTJgwgcGDB1e4j5YtW5JIJBgyZAgdO3Ysm3Sobdu2XHzxxXz7299m\n27ZtHDhwgKKiIubNmwfAV7/6VR544AGKi4v58MMPueuuu2qs3enoCW4RkSS7d+/mlltuoUWLFrRp\n04YNGzZw5513Mm7cOHr06EHXrl0566yz6NatG+PGjat0P4MGDWLu3Llll6BK/epXv2LPnj106dKF\npk2bctVVV/H+++8DMHToUPr06cOnP/1punXrxpe//OVY21oVmla1Fk47GTe1OX/UxnZrWtXqp2lV\nRUSkVlCwEBGRtBQsREQkLQULERFJS8FCRGpcrtxYc7Sojp+3goWI1Kh69eqxa9eubFcjr+zduzfl\n2FKZULAQkRrVqlUr1q5dWzb4nsTrwIED/Oc//6Fx48ZHtJ9sjzorInmmUaNGQBgHae/evdW+/48+\n+oj69etX+35rs3RtPuGEE2jRosURHUPBQkRqXKNGjcqCRnUrLCzknHPOiWXftVVNtFmXoUREJC0F\nCxERSUvBQnJWURGMGgWtWsEXvtCbVq1Cuqgo/bbVeey6danRY4vEQcFCctLs2dCzJ5SUwKRJMH++\nMWlSSPfsGdbX3LGpsWOLxEUd3JJziopg8GC45x7o2vVgfvv2MHw49OoV1i9YAAUFuXNskTjpzEJy\nzgMPQP/+h35ZJ+vaNax/8MHcOrZInBQsJOdMmxa+kFPp3x+mTs2tY4vEScEiT2Szs7embdoEbdqk\nLtOmDWzenFvHzjZ16uc2BYs8kM3OXqj5L5HmzWH9+tRl1q+HZs1y69jZpE793KdgkeOSO1yHDw8d\nrcccc7DD9Z57wvq4vriz8SUycCDMnJm6zMyZUG5q5KP+2NmS7c+Y1AwFixyXzQ7XbH2JjBwZvpCX\nLKl4/ZIlYf2NN1bvcbN97GxRp35+ULDIcdnscM3Wl0hBAUyZAmPGwMSJUFwM+/aF14kTQ/6UKfHc\nuprNY2eLOvXzg4JFjstmh2s2v0T69QvPMjRqBEOHwvnnO0OHhvSCBWF9XD5+bGrs2NmQz536+UQP\n5eW40g7X9u0rLxNXh2u2v0QKCuC++8JSWDiPRCIRz4HSHDvXZfMzJjUn1jMLM+trZsvNbKWZ3VJJ\nma+a2TIzW2pmU5PyrzGz96Llmjjrmcuy2eGar3cG5Zt87NTPR7EFCzOrC0wE+gFdgIFm1qVcmc7A\nWOB8dz8TGB3lNwN+AJwL9AR+YGZN46prLstmh6u+RPJDPnbq56M4zyx6AivdfZW77wGmA+WvYA8F\nJrr7hwDuviHK7wO85O6bo3UvAX1jrGvOymaHq75EsqOmH8DMx079fBRnsGgHrElKF0d5yU4DTjOz\n18zsdTPrW4VtJUPZ6uzVl0jNy9YDmPnWqZ+PLK4J083sKqCvu38zSn8dONfdRySV+QOwF/gq0B74\nM3AW8E2gvrvfHpX7PrDL3e8pd4xhwDCA1q1bd58+fXqV67l9+3ZOPPHEqjfwKFbTbV67tj7PPtue\nl19uw9atdWnceD+f//x6Lr+8mHbtPqqROuTD73nt2vqMGPEZ7r23boW3Ky9ZAjffvJ+HHlpYYz/3\nbMiH33V5R9LmCy644E1375GuXJx3Q60FOiSl20d5yYqBN9x9L/BPM1sBdI7KJcptW1j+AO7+GPAY\nQI8ePfxw7nYpLCys0btkaoNstPnqq5NTxxB+pSlun6lm+fB7HjUKrrgi9XMtV1xRl4ULzyv3+8gt\n+fC7Lq8m2hznZaiFQGcz62hmxwIDgFnlyjxLFBTMrAXhstQqYA5wsZk1jTq2L47yRKQSejhO4hTb\nmYW77zOzEYQv+brAZHdfamYTgEXuPouDQWEZsB/4jrtvAjCz2wgBB2CCu+uRHpEUsv1ci+S2WJ+z\ncPcX3P00dy9w9zuivPFRoMCDm929i7uf5e7Tk7ad7O6fiJbH46ynSC7Qcy01L5+GZddwHyI5Ip+f\na8nGfC35Niy7goVIjsjX51qycbtwPg7LrrGhRHJE6XMtgweHjuz+/UMfxfr1IUjMnJl7z7Ukf2kn\n3wVW+qXdq1dYv2BB9ba7KiMq58r4YDqzEMkh2RxtNxuyNQx+Pt55pmAhkmNKR7zdsAHmzp3Hhg0h\nnUtnFKWy9aWdj3eeKViIyFErW1/a+XjnmYKFiBy1svWlnY93nilYiMhRK1tf2vl451lsAwnWtIYN\nG3r37t2rvN2WLVto0qRJDDWqvdTm/JHr7d61C/72t9Afc8IJH1+/Y0e4Y6pbN2jQoHqPvXkzvPsu\ntGgRznCOPRb27AmXxjZuhDPOqLnLUEfye543b17WBxIUEYlVgwbhSzndl3Z1BwoIgaBbN1i7Flas\ngL17oV698GBgHMEp23ImWJx++ukUFhZWeTuNUJkf8rHNkD/tLioKt8dOnQqbNzvNmhmDBoXLQLl4\nF1h5R/J7NrOMyqnPQkSOevl0u3Cpmh7iRMFCROQok40hThQsalg+jVIpItUvW+NSpQ0WZnZW9R4y\nf+XbKJUiUv2yNcRJJmcWD5vZAjP7lpk1rt7D5498HKVSRKpftoY4SRss3L0XcDVhPu03zWyqmV1U\nvdXIfdn6b0BEcku2hjjJqM/C3d8DxgHfBXoDD5jZP8zsy9VbndyVj6NUikj1y9YQJ5n0WXQ1s58B\n7wKfB77k7mdE739WvdXJXfk4SqWIVL9sDXGSyUN5DwK/AL7n7rtKM919nZmNq97q5K7S/wbat6+8\nTK6NUiki1W/kyHBDTK9eFV/WLh2XasGC6j1uJpehLgGmlgYKM6tjZscDuPuvq7c6uSsfR6kUkepX\nOiPimDEwcSIUF8O+feF14sSQH8eMiJkEi7lA8ignx0d5UgX5OEqliMQjGzMiZnIZqr67by9NuPv2\n0jMLyVw+zo8sIvEpHeLkvvugsHBe7GOAZXJmscPMupUmzKw7sCtFeanEx/8bIKfnRxaR3JHJmcVo\n4Ldmtg4woA3wtVhrlcOS/xsQETlaZPJQ3kLgk8D/ADcAZ7j7m5ns3Mz6mtlyM1tpZrdUsP5aM/vA\nzBZHyzeT1u1Pyp+VeZNERKS6ZTqfxelAF6A+0M3McPdfpdrAzOoCE4GLgGJgoZnNcvdl5YrOcPcR\nFexil7ufnWH9REQkRmmDhZn9AEgQgsULQD/gVSBlsAB6AivdfVW0n+lAf6B8sBARkVoukw7uq4Av\nAOvdfQjwaSCTAQXbAWuS0sVRXnlXmtkSM/udmXVIyq9vZovM7HUzuzyD44mISEwyuQy1y90PmNk+\nM2sEbCAMKlgdngOmuftuM7seeJIwjAjAKe6+1sw6AS+b2dvufsiYrGY2DBgG0Lp168OaVnX79u2H\ntd3RTG3OH/nYbrU5HpkEi0Vm1gSYBLwJbAf+msF2azk0qLSP8sq4+6ak5C+AnyStWxu9rjKzQuAc\noKjc9o8BjwH06NHDD+c+43yZoziZ2pw/8rHdanM8Ul6GsjCT953uvsXdHyV0Vl8TXY5KZyHQ2cw6\nmtmxwADgkLuazKxtUvIywmCFmFlTMzsuet8COB/1dYiIZE3KMwt3dzN7ATgrSv8r0x27+z4zGwHM\nAeoCk919qZlNABa5+yxgpJldBuwDNgPXRpufAfzczA4QAtpdFdxFJSIiNSSTy1B/M7PPRM9bVIm7\nv0C4gyo5b3zS+7HA2Aq2m08UoEREJPsyCRbnAleb2WpgB+Epbnf3SuZ8ExGRXJPJrbN9gAKiiY+A\nS6PXvFRUBKNGQatWULdueB01SnNni0huyyRYeCVL3pk9O0w6UlICkybB/PnhtaQk5M+ene0aiojE\nI5PLUM8TgoMRhvvoCCwHzoyxXrVOUVEYXvyeew6dnap9exg+PMxaNXhwGD1Ww4yLSK7JZCDBs9y9\na/TamTCMRybPWeSUBx4Ic1BUNI0hhPz+/eHBB2u2XiIiNSGTy1CHcPe/ETq988q0aeFS0113Qe/e\n8JnPhNe77grTGUIIFlOnZreeIiJxyGQgwZuTknWAbsC62GpUS23cCDfdFOa63bcv5O3YAc8+C3/4\nA9x9N5x7LmzenN16iojEIZMzi4ZJy3GEPoz+cVaqtnn5ZXCHjz46GChK7dsX8r/7XXjrLWjWLDt1\nFBGJU9ozC3f/UU1UpDYbPTp9mX374Gc/g0GD4q+PiEhNS3tmYWYvRQMJlqabmtmceKtVu7zzTvoy\n+/bBihVw443x10dEpKZlchmqpbtvKU24+4dAq/iqVPt4FZ4q0W2zIpKLMgkW+83s5NKEmZ1Cnj6U\nl45ZtmsgIhKPTB7KuxV41czmER7M60U04VA+ePnlMKzHgQOpzzDM4CwNfSgiOSqTh/JeJNwuOwOY\nDnR391rXZ7F8+XKeeOIJAPbu3UsikWDKlCkA7Ny5k0QiwYwZMwDYunUriUSCp59+GoCNGzeSSCR4\n7rnnAFi/fj2JRILbbnuRyy6Dz352De4JYG50tFWEacnnlR4d9wTf/OZ8AN555x0SiQQLF4aBehcv\nXkwikWDx4sUALFy4kEQiwTtRZ8j8+fNJJBIsX74cgHnz5pFIJFi1ahUAc+fOJZFIsGZNmKX2xRdf\nJJFIsH79egCee+45EokEGzduBODpp58mkUiwdetWAGbMmEEikWDnzp0AvPTSSyQSCfbu3QvAE088\nccjEKZMmTeLCCy8sSz/88MP069evLH3//fdz2WWXlaXvuecerrzyyrL0XXfdxYABA8rSt912G4MH\nDy5Ljx8/niFDDk6JMnbsWIYNO/j/x5gxYxg+fHhZevTo0YxOustg+PDhjBkzpiw9bNgwxo49OHjx\nkCFDGD++bHBjBg8ezK9+dXDK+AEDBnDXXXeVpa+88kruueeesvRll13G/fffX5bu168fDz/8cFn6\nwgsvZNKkSWXpRCJR7Z+9F198EYA1a9aQSCSYOzd89latWkUikWDevPDZW758OYlEgvnzK/7srVy5\nslZ99qZMmRL7Z2/ChAll6drw2bvtttvK0rXxs5epTDq4rwD2uvsf3P0PwL58mBN71y74yU/C2cQ1\n10DbaJqm8peaStNm0L17zdZRRKTGuHvKBVhcQd5b6bar6aV79+5+OF555ZUK80eOdB8yxN3M/fXX\n3Rctcn/kEffOnUMehNfOnd0feii8P1pU1uZclo9tds/PdqvNVUOYjC7td2wmfRYVnX1kst1Rbdo0\nmDAhDN+xfn0YMPAznwn55RUXw7HH1nwdRURqSiZ3Qy0ys3vNrCBa7gXejLti2bZpE8ybFzq3n302\nddlnnlGwEJHclkmwuBHYQ+jgngHsBoan3CIHNG8OL74I550Hv/sdLFlScbklS8L6L3+5ZusnIlKT\nMrkbaoe73+LuPaJlrLvvqInKZdPAgbB1K3zrW6GTe/RoeOihcMlp377w+tBDIX/PHvj+97NdYxGR\n+GQy6mxL4H8Jkx3VL81398/HWK+sGzkSfv5zOOYY+PGPYdw4WLQIZs6EbdugUSNo1y48f3HMMXpy\nW0RyWyaXoZ4C/kGYIe9HwL+AhTHWqVYoKAhB4Nln4fzz4de/DhMc1Yl+YnXqhPRFF4VyIiK5LJNg\n0dzdf0l41mKeu18H5PRZRamdO0OwWLIk3A11880wZw688UZ4vfBCeOWVMK+FiEguyyRY7I1e3zez\nS8zsHCAvZm1o0SKMInvzzRX3V9x8c1jfvHm2ayoiEq9MgsXtZtYY+DYwBvgFcFOstaolLrkE/vlP\neOKJ0In9jW+ES1Lf+EZIP/EErFoFl16a7ZqKiMQrk7uh/uDuW939HXe/wN27u/usTHZuZn3NbLmZ\nrTSzWypYf62ZfWBmi6Plm0nrrjGz96Llmqo1q3ocOBAuQ23e/PFLUDffHPJnzgzlRERyWWxds2ZW\nF5gIXAQUAwvNbJa7LytXdIa7jyi3bTPgB0APwnDob0bbfhhXfSsye3aYd/vmm+Hyy8PSpk14ovvZ\nZ8Ny003w6KM1WSsRkZqXyWWow9UTWOnuq9x9D2HE2kzn7u4DvOTum6MA8RLQN6Z6VmrTpnApKtVl\nqEsuCWcYIiK5LM6bPtsBa5LSxcC5FZS70sw+B6wAbnL3NZVs2y6uilamefOD40LdfHNYyisuhmZ5\n0d0vIvksk4fyjgOuBE5NLu/uEyrbpgqeA6a5+24zux54kirclmtmw4gmYmrdujWFhYVVrsD27dsr\n3a537848+2xbRoyo/ATsmWcO0Lv3OgoLV1b52NmSqs25Kh/bDPnZbrU5HpmcWcwEthIGD9xdhX2v\nBTokpdtHeWXcfVNS8hfAT5K2TZTbtrD8Adz9MeAxgB49enhVJvIoVVhYWOkEIB06QM+e8LnPhQfw\nyluyBJ57rg4LFrSnoKB9lY+dLananKvysc2Qn+1Wm+ORSbBo7+6H01+wEOhsZh0JX/4DgEHJBcys\nrbu/HyUvA96N3s8BfmxmTaP0xcBYalhBAUyZAoMHQ//+YSnt4J45MyxTpmioDxHJfZkEi/lmdpa7\nv12VHbv7PjMbQfjirwtMdvelZjaBMNnGLGCkmV0G7AM2A9dG2242s9s4OKzIBHfPSjdyv36wYAE8\n+CAMHRo6s5s1g0GDQr4ChYjkg0yCxX8D15rZPwmXoQxwd6/gwsyh3P0F4IVyeeOT3o+lkjMGd58M\nTM6gfrErKID77guLiEg+yiRY9EtfJLcUFcEDD4RZ8TZtCndFDRwYRqLVmYSI5KNMnuBeDTQBvhQt\nTaK8nDR7dujULimBSZNg/vzwWlIS8mfPznYNRURqXia3zo4ChgJPR1lTzOwxd38w1pplQVFR6My+\n555D735q3x6GD4devcJ69VWISL7J5DLUN4BzS2fHM7O7gb8CORcsHngg3PHUtWt42G7GjDAO1JYt\n0KQJ9OkDF1wQOrvVfyEi+SST4T4M2J+U3h/l5Zxp00KweO01uPZaqF8fJk8Ol6ImTw7puXPh8cez\nXVMRkZqVyZnF48AbZvZMlL4c+GV8VcqeTZvCfBU/+AHce2/ll6K+9a1wyUqXokQkX2TSwX0vMITw\nHMRmYIjsNFyCAAAVsUlEQVS75+RFmObN4ckn4YorKn5iG0L+lVeGS1EiIvmi0mBhZo2i12aEeben\nRMvqKC/nDBwYpkntn2Zs3K98BaZOrZk6iYjUBqnOLEq/Dt8EFiUtpemcM3JkmE+7TZvU5dq00bDk\nIpJfKu2zcPdLo9eONVed7CooCHc9lQ5LXpn16zUsuYjkl7R9Fmb2p0zycsXXvx4GCExl5swwNpSI\nSL6o9MzCzOoDxwMtotFfS2+XbUQWJiKqKSNHhie1e/WqfFjymTPDg3kiIvki1a2z1wOjgZMI/RSl\nwWIb8FDM9coaDUsuIvJxqfos7gfuN7Mbc3Foj1Q0LLmIyKHSPpTn7g+a2aeALkD9pPxfxVmxbNOw\n5CIiB2UykOAPCFOcdiHMTdEPeBXI6WAhIiIHZTI21FXAF4D17j4E+DTQONZaZUlREYwaBa1aQd26\n4XXUqJAvIpLPMgkWu9z9ALAveqp7A9Ah3mrVPM1jISJSuUwGElxkZk2ASYS7orYThijPGZrHQkQk\ntUwGEvyWu29x90eBi4BrostROSN5HouKdO0a1mvwQBHJV6kGEuxWfgGaAcdE73NG6TwWqfTvr8ED\nRSR/pboM9dPotT7QA/g74cG8roSBBD8bb9VqzqZNGjxQRCSVSs8s3P0Cd78AeB/o5u493L07cA6w\ntqYqWBOaNw9PaKeiwQNFJJ9lcjfU6e7+dmnC3d8BzoivSjVv4EANHigikkomd0MtMbNfECY+Arga\nWBJflWqeBg8UEUktkzOLIcBSYFS0LIvy0jKzvma23MxWmtktKcpdaWZuZj2i9KlmtsvMFkfLo5kc\n73AVFMD//V8IGj/7GRQXh7m4i4vhoYdgzBgNHigi+S2TsaE+An4WLRkzs7rARMLttsXAQjOb5e7L\nypVrSAhCb5TbRZG7n12VYx6u2bPhO9+BPn3CQ3jXXQdbtsDxx4f1998fBhcUEclXqeaz+I27f9XM\n3ga8/Hp3r+SphDI9gZXuvira33SgP+HMJNltwN3Ad6pS8eqydm19Ro78+AN5pZYsCWcWvXvrzEJE\n8leqy1CjotdLgS9VsKTTDliTlC6m3KRJ0fMaHdz9+Qq272hmb5nZPDPrlcHxDsuzz3bQA3kiImmk\nms/i/eh1dRwHNrM6wL3AtRWsfh842d03mVl34FkzO9Pdt5XbxzBgGEDr1q0pLCyscj3mzj2fJ55I\nXaZ/f7juun1cfvmrVd5/bbR9+/bD+lkdzfKxzZCf7Vab45HqMlQJFVx+IjyY5+7eKM2+13LogIPt\nOfT5jIbAp4BCMwNoA8wys8vcfRGwm3CgN82sCDiN8DBgGXd/DHgMoEePHp5IJNJU6eO2bfOMHsjb\nuvUYDmf/tVFhYWHOtCVT+dhmyM92q83xSHVm0fAI970Q6GxmHQlBYgBQ9qSCu28FWpSmzawQGOPu\ni8ysJbDZ3febWSegM7DqCOtTocaN97N+/TG0b195GT2QJyL5LpNbZwEws1ZmdnLpkq68u+8DRgBz\ngHeB37j7UjObYGaXpdn8c4TnOxYDvwNucPdYBtv4whf+owfyRETSyGSmvMsI40SdRJjL4hTCl/+Z\n6bZ19xcIs+sl542vpGwi6f3vgd+n2391uPzyNYwc2U4P5ImIpJDJE9y3AecBc939HDO7ABgcb7Vq\nTrt2HzFlSpivon//sLRpEy49zZwZFj2QJyL5LpPLUHvdfRNQx8zquPsrhFFoc0a/fuHMoVEjGDoU\nzj8/vDZqFPL1QJ6I5LtMziy2mNmJwJ+Bp8xsA7Aj3mrVvIICuO++sIiIyKEyObPoD+wCbgJeBIrI\n7KE8ERHJEames5gITHX315Kyn4y/SiIiUtukOrNYAdxjZv8ys5+Y2Tk1VSkREaldUs2Ud7+7fxbo\nDWwCJpvZP8zsB2Z2Wo3VUEREsi5tn4W7r3b3u939HGAgcDnhOQsREckTaYOFmR1jZl8ys6eA2cBy\n4Mux10xERGqNVB3cFxHOJL4ILACmA8PcPedumxURkdRSPWcxFpgKfNvdP6yh+oiISC2UatTZz9dk\nRUREpPbKeNRZERHJXwoWIiKSloKFiIikpWAhIiJpKViIiEhaChYiIpKWgoWIiKSlYCEiImkpWIiI\nSFoKFiIikpaChYiIpKVgISIiaSlYiIhIWrEGCzPra2bLzWylmd2SotyVZuZm1iMpb2y03XIz6xNn\nPUVEJLVU81kcETOrC0wELgKKgYVmNsvdl5Ur1xAYBbyRlNcFGACcCZwEzDWz09x9f1z1FRGRysV5\nZtETWOnuq9x9D2Gmvf4VlLsNuBv4KCmvPzDd3Xe7+z+BldH+REQkC+IMFu2ANUnp4iivjJl1Azq4\n+/NV3VZERGpObJeh0jGzOsC9wLVHsI9hwDCA1q1bU1hYWOV9bN++/bC2O5qpzfkjH9utNscjzmCx\nFuiQlG4f5ZVqCHwKKDQzgDbALDO7LINtAXD3x4DHAHr06OGJRKLKlXzqqdd56aXzmDYNNm2C5s1h\n4EAYORIKCqq8u6NCYWEhh/OzOprlY5shP9utNscjzstQC4HOZtbRzI4ldFjPKl3p7lvdvYW7n+ru\npwKvA5e5+6Ko3AAzO87MOgKdgQXVXcHZs2HEiM9QUgKTJsH8+eG1pAR69gzrRUQkxjMLd99nZiOA\nOUBdYLK7LzWzCcAid5+VYtulZvYbYBmwDxhe3XdCFRXB4MFw77116dr1YH779jB8OPTqFdYvWJC7\nZxgiIpmKtc/C3V8AXiiXN76Ssoly6TuAO+Kq2wMPQP/+HBIoknXtGtY/+CDcd19ctRAROTrk7RPc\n06aFYJBK//4wdWrN1EdEpDbL22CxaRO0aZO6TJs2sHlzzdRHRKQ2y9tg0bw5rF+fusz69dCsWc3U\nR0SkNsvbYDFwIMycmbrMzJkwaFDN1EdEpDbL2kN52TZyZLg9tleviju5lywJwWJBtd+wKyJy9Mnb\nYFFQAFOmwKBB+7niirr07x/6KNavD0Fi5sywXrfNiojkcbAA6NcPHnpoIQsXnsfQoaEzu1mzcOlJ\nz1eIiByU18ECoF27j7j6aj1LISKSSt52cIuISOYULEREJC0FCxERSUvBQkRE0lKwEBGRtBQsREQk\nLQULERFJS8FCRETSUrAQEZG0FCxERCQtBQsREUlLwUJERNJSsBARkbQULEREJC0FCxERSUvBQkRE\n0lKwEBGRtGINFmbW18yWm9lKM7ulgvU3mNnbZrbYzF41sy5R/qlmtivKX2xmj8ZZTxERSS22aVXN\nrC4wEbgIKAYWmtksd1+WVGyquz8alb8MuBfoG60rcvez46qfiIhkLs4zi57ASndf5e57gOlA/+QC\n7r4tKXkC4DHWR0REDlOcwaIdsCYpXRzlHcLMhptZEfATYGTSqo5m9paZzTOzXjHWU0RE0jD3eP6Z\nN7OrgL7u/s0o/XXgXHcfUUn5QUAfd7/GzI4DTnT3TWbWHXgWOLPcmQhmNgwYBtC6devu06dPr3I9\nt2/fzoknnljl7Y5manP+yMd2q81Vc8EFF7zp7j3SFnT3WBbgs8CcpPRYYGyK8nWArZWsKwR6pDpe\n9+7d/XC88sorh7Xd0Uxtzh/52G61uWqARZ7Bd3qcl6EWAp3NrKOZHQsMAGYlFzCzzknJS4D3ovyW\nUQc5ZtYJ6AysirGuIiKSQmx3Q7n7PjMbAcwB6gKT3X2pmU0gRLJZwAgzuxDYC3wIXBNt/jlggpnt\nBQ4AN7j75rjqKiIiqcUWLADc/QXghXJ545Pej6pku98Dv4+zbiIikjk9wS0iImkpWIiISFoKFiIi\nkpaChYiIpKVgISIiaSlYiIhIWgoWIiKSloKFiIikpWAhIiJpKViIiEhaChYiIpKWgoWIiKSV18Gi\nqAgmTuxMq1ZQty60agWjRoV8ERE5KG+DxezZ0LMnnHhiWyZNgvnzYdIkKCkJ+bNnZ7uGIiK1R6xD\nlNdWRUUweDDccw907XowXrZvD8OHQ69eYf2CBVBQkMWKiojUEnl5ZvHAA9C/P3TtWvH6rl3D+gcf\nrNl6iYjUVnkZLKZNC8Eglf79YerUmqmPiEhtl5fBYtMmaNMmdZk2bWCzJnIVEQHyNFg0bw7r16cu\ns349NGtWM/UREant8jJYDBwIM2emLjNzJgwaVDP1ERGp7fLybqiRI8Ptsb16VdzJvWRJCBYLFtR8\n3UREaqO8DBYFBTBlSrg99ktfOsAVV9ShTZtw6WnmzLBMmaLbZkVESuXlZSiAfv3CmcOOHesYOhTO\nPx+GDoVGjUJ+v37ZrqGISO2Rl2cWpQoKYPjwlfz2t+2zXRURkVotb88sREQkc7EGCzPra2bLzWyl\nmd1SwfobzOxtM1tsZq+aWZekdWOj7ZabWZ846ykiIqnFFizMrC4wEegHdAEGJgeDyFR3P8vdzwZ+\nAtwbbdsFGACcCfQFHo72JyIiWRDnmUVPYKW7r3L3PcB04JBBNtx9W1LyBMCj9/2B6e6+293/CayM\n9iciIlkQZwd3O2BNUroYOLd8ITMbDtwMHAt8Pmnb18tt2y6eaoqISDpZvxvK3ScCE81sEDAOuCbT\nbc1sGDAsSm43s+WHUYUWwMbD2O5opjbnj3xst9pcNadkUijOYLEW6JCUbh/lVWY68EhVtnX3x4DH\njqSSZrbI3XscyT6ONmpz/sjHdqvN8Yizz2Ih0NnMOprZsYQO61nJBcysc1LyEuC96P0sYICZHWdm\nHYHOgAbfEBHJktjOLNx9n5mNAOYAdYHJ7r7UzCYAi9x9FjDCzC4E9gIfEl2Cisr9BlgG7AOGu/v+\nuOoqIiKpxdpn4e4vAC+Uyxuf9H5Uim3vAO6Ir3Zljugy1lFKbc4f+dhutTkG5u7pS4mISF7TcB8i\nIpJWXgeLdMOR1HZmNtnMNpjZO0l5zczsJTN7L3ptGuWbmT0QtXWJmXVL2uaaqPx7ZnZNUn73aDiW\nldG2VrMt/Dgz62Bmr5jZMjNbamajovycbbeZ1TezBWb296jNP4ryO5rZG1E9Z0Q3khDdGDIjyn/D\nzE5N2leFw+jU1r8FM6trZm+Z2R+idE632cz+ZQeHQFoU5dWOz7a75+VC6HQvAjoRHgj8O9Al2/Wq\nYhs+B3QD3knK+wlwS/T+FuDu6P0XgdmAAecBb0T5zYBV0WvT6H3TaN2CqKxF2/arBW1uC3SL3jcE\nVhCGk8nZdkf1ODF6Xw94I6rfb4ABUf6jwP9E778FPBq9HwDMiN53iT7nxwEdo89/3dr8t0B4YHcq\n8IcondNtBv4FtCiXVys+2/l8ZpF2OJLazt3/DGwul90feDJ6/yRweVL+rzx4HWhiZm2BPsBL7r7Z\n3T8EXgL6RusaufvrHj5lv0raV9a4+/vu/rfofQnwLuHp/pxtd1T37VGyXrQ4YcSD30X55dtc+rP4\nHfCF6D/IyobRqZV/C2bWnnBL/S+itJHjba5Erfhs53OwqGg4klwYUqS1u78fvV8PtI7eV9beVPnF\nFeTXGtGlhnMI/2nndLujyzGLgQ2EP/4iYIu774uKJNezrG3R+q1Ac6r+s8i2+4D/BQ5E6ebkfpsd\n+KOZvWlhhAqoJZ/trA/3IfFxdzeznLzdzcxOBH4PjHb3bcmXXnOx3R6eMzrbzJoAzwCfzHKVYmVm\nlwIb3P1NM0tkuz416L/dfa2ZtQJeMrN/JK/M5mc7n88sqjocydHiP9HpJtHrhii/svamym9fQX7W\nmVk9QqB4yt2fjrJzvt0A7r4FeAX4LOGyQ+k/fMn1LGtbtL4xsImq/yyy6XzgMjP7F+ES0eeB+8nt\nNuPua6PXDYR/CnpSWz7b2e7QydZCOKtaRej0Ku3gOjPb9TqMdpzKoR3c/8ehnWE/id5fwqGdYQv8\nYGfYPwkdYU2j98284s6wL9aC9hrhWut95fJztt1AS6BJ9L4B8BfgUuC3HNrZ+63o/XAO7ez9TfT+\nTA7t7F1F6Oit1X8LQIKDHdw522bCNA0Nk97PJ8znUys+21n/IGT5l/NFwt00RcCt2a7PYdR/GvA+\nYbiUYuAbhOu0fyKMszU36UNihMmoioC3gR5J+7mO0PG3EhiSlN8DeCfa5iGihziz3Ob/JlzXXQIs\njpYv5nK7ga7AW1Gb3wHGR/mdoj/+ldGX6HFRfv0ovTJa3ylpX7dG7VpO0p0wtflvgUODRc62OWrb\n36NlaWmdastnW09wi4hIWvncZyEiIhlSsBARkbQULEREJC0FCxERSUvBQkRE0lKwEBGRtBQsREQk\nLQULyRlmdms038OSaD6Ac1OUPdWS5gGpzczsh2Y2Jno/v5r3/YtoHCaRlDSQoOQEM/ssYQiMbu6+\n28xaEIZxyCnu/l/VvMtzgB9W8z4lB+nMQnJFW2Cju+8GcPeN7r6u/BmEmY0xsx9GyWPM7Ckze9fM\nfmdmx0dlTjCz5y3MTPeOmX0t2s8/Kin/bDSk9NKkYaUxs/8XneX83cx+nZQ/2MLMd4vN7OdmVrd8\nY6KzpBVm9ipwelL+9ui1tD5PROWeMrMLzey1aHa0nhX9kMzsNDN7NZot7VagjbsXV1RWJJmCheSK\nPwIdoi/Oh82sdwbbnA487O5nANsIs61BGLxtnbt/2t0/BbyYpvx17t6dMO7OSDNrbmZnAuOAz7v7\np4HS6V/PAL4GnO/uZwP7gauTK2Vm3QmD4Z1NGL/oM5XU/xPATwnDlX8SGEQYO2sM8L3yhc3sOMJI\npje7+1mEuQz+Ub6cSEUULCQneJhJrjswDPgAmGFm16bZbI27vxa9n0L4ooUwKNtFZna3mfVy961p\nyo80s78DrxOGhu5MGFL7t+6+Mapf6YyGX4jquTCazOgLhAHkkvUCnnH3ne6+DZhVSf3/6e5vu/sB\nwsBzf/Iw2NvbhNGIy7scWOTuC6L0UsKgdSJpqc9CcoaHCYIKgUIzexu4hjBKZ/I/RfWTNym/i2g/\nK8ysG+G/+tvN7E+EYdE/Vj6amOdC4LPuvtPMCssdozwDnnT3sVVoWmV2J70/kJQ+QMV/22cBbyal\nuxN+XiJp6cxCcoKZnW5mnZOyzgZWA/8BWkWXho4jdIKXOjnqGIdwCefVaF8nATvdfQphLoFuKco3\nBj6MAsUnCXMFALwMfMXMmkf7bBbl/wm4KpoJDTNrZmanlGvOn4HLzayBmTUEvnQ4P5MKbAI+FR23\nOzAQnVlIhnRmIbniRODBaNrRfYRx/Ie5+14zm0CY42Ath16jXw4MN7PJwDLgkSj/LOD/zOwAYa6Q\n/0lRfj9wg5m9G61/HcDdl5rZHcA8M9tPmI/iWndfZmbjCPMs14n2P5wQ2Ii2/ZuZzSB8kW8AFlbT\nz+jXwAvR5a/lwJaoHSJpaT4LkQyY2amECXg+leWqiGSFLkOJiEhaOrMQEZG0dGYhIiJpKViIiEha\nChYiIpKWgoWIiKSlYCEiImkpWIiISFoKFiIikpaChYiIpPX/Adz9elHNfHyAAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2fcbbbbfd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = subplots(figsize=(6,5) )\n",
    "  \n",
    "font = {'size'   : 12}\n",
    "matplotlib.rc('font', **font)\n",
    "\n",
    "\n",
    "plot(dim[1:], Rs[1:,1], 'o', mec='b', mfc=(.8,.8,1), ms=10)\n",
    "plot(dim_solved, acc_solved, 'o', mec='b', mfc='b', ms=10)\n",
    "axhline(Rs[0,1], ls='-', color='k',label='baseline')\n",
    "axhline(Rs[0,1] * .9, ls=':', color='k',label='solved')\n",
    "plt.legend()\n",
    "ax.set_xlabel('Subspace dim $d$')\n",
    "ax.set_ylabel('Validation accuracy')\n",
    "\n",
    "# ax.set_title('width %d, depth %d' %(width[i], depth[j]))\n",
    "plt.grid()\n",
    "ax.set_ylim([0.3,0.7])\n",
    "        \n",
    "fig.savefig(\"figs/lenet_cifar_dim.pdf\", bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above figure show that updating in the intrinsic space can prevent overfitting.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUsAAAEQCAYAAAA5/laqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGdVJREFUeJzt3X983VV9x/HXJ23XIkmFigtTeLRFBdLUgQTm2CI2g40f\nArrhlCBz7eKqbmR2wpwzurVCKGo3fxRUwJQyS5I5wQrq1G1Nt0Wn2LoxCwGUH4UKRWkhNIW2sf3s\nj3Ov3lzb5txf3/u9N+/n43Ef7f3e7/l+z2l63znfX+eYuyMiIofXUO0KiIjUAoWliEgEhaWISASF\npYhIBIWliEgEhaWISASFpYhIBIWliEgEhaWISITp1a7A4RxzzDE+b968qHV3797NkUceWdkKJURt\nSSe1JZ2KacvmzZufdveXFlIm1WE5b948Nm3aFLXuxo0bWbRoUWUrlBC1JZ3UlnQqpi1mtrXQ/egw\nXEQkgsJSRCSCwlJEJILCUkQkgsJSRCSCwlJEJILCUkQkQqrvsyxKzP1WF14IV131i/UXLw6vp5+G\nN7958vL56195JVx0ETzwALzznZOXz1//2mvht34Lvv1t+MAHOPXZZ+Goow5dPm99brwRTjoJ7roL\n/v7vJ99//vpf/CIccwysXRtek8lff+PGsHzVKvjKVyasetC25K7/3/8Nt98e3v/N34T3h/OSl0xc\nf8cOuOmm8H7pUnjwwcOXP/HEieu/5CWwcmV4f8klYXuHcOqzz8L5509c/8wzJ/5fmkxK/u8d8dhj\nYf/5/5cmk9b/ewlQz1JEJIa7p/bV1tbmsYaGhqLXTTu1JZ3UlnQqpi3AJi8wj9SzFBGJoLAUEYmg\nsBQRiaCwFBGJoLAUEYmgsBQRiaCwFBGJoLAUEYmgsBQRiaCwFBGJoLAUEYmgsBQRiaCwFBGJoLAU\nEYmgsBQRiaCwFBGJoLAUEYmgsBQRiZBYWJrZRjPbY2ZjmdcDSe1bRKRUSfcsr3D3xszrpIT3LSJS\nNB2Gi4hESDosV5rZ02b2LTNblPC+RUSKZmFWyAR2ZPZa4D5gH3ApcD1wqrs/lLfeUmApQHNzc9vg\n4GDU9sfGxmhsbCxrnatFbUkntSWdimlLR0fHZnc/vaBChc6dW64X8HWg+3DraN7w2qe2pNNUbws1\nNm+4A1bF/YuIREskLM3sKDM718xmmdl0M3sbcBahdykiknrTE9rPDOAa4GRgP3A/8CZ3fzCh/YuI\nlCSRsHT3nwJnJLEvEZFK0H2WIiIRFJYiIhEUliIiERSWIiIRFJYiIhEUliIiERSWIiIRFJYiIhEU\nliIiERSWIiIRFJYiIhEUliIiERSWIiIRFJYiIhEUliIiERSWIiIRFJYiIhEUliIiERSWIiIRFJYi\nIhEUliIiERSWIiIRFJYiIhEUliIiERSWIiIRFJYiIhEUliIiERSWIiIRFJYiIhEUliIiERSWIiIR\nFJYiIhEUliIiERSWIiIRFJYiIhESD0sze5WZ7TGzdUnvW0SkWNXoWd4AfK8K+xURKVqiYWlmlwLP\nAv+e5H5FREqVWFia2Wzgw8B7k9qniEi5mLsnsyOzTwJPuPtHzGw58Ep3v/wg6y0FlgI0Nze3DQ4O\nRm1/bGyMxsbGMta4etSWdFJb0qmYtnR0dGx299MLKuTuFX8BpwL3Ar+Seb8cWDdZuba2No81NDQU\nvW7aqS3ppLakUzFtATZ5gTk2vaBkLd4iYB7wmJkBNALTzGyBu5+WUB1ERIqWVFjeBOQeT19FCM93\nJ7R/EZGSJBKW7v488Hz2vZmNAXvc/adJ7F9EpFRRYWlm7wFuc/eny7FTd19eju2IiCQl9tah3wEe\nNbOvmNlbzWxmJSslIpI2UWHp7m8E5gL/AiwDtpvZ58zsrEpWTkQkLaJvSnf3He5+g7ufCbweOAMY\nMrNHzazHzOrjpi0RkYMo6AkeMzvbzG4BNgJPAW8H/gh4DaHXKSJSl2Iv8KwCLgVGgX8EPujuP875\n/DvAMxWpoYhICsTeOjQL+H13P+hoQe4+bmaFPTokIlJDYsNyJTn3SQKY2dHAEe7+BIC731/muomI\npEbsOcv1wHF5y44DvlTe6oiIpFNsWJ7k7j/IXZB5f3L5qyQikj6xYfkTM3tl7oLM+x3lr5KISPrE\nhuUa4HYzu9DMFpjZRcAXgc9VrmoiIukRe4HnOmAcWAUcDzxOCMp/qFC9RERSJSos3f0A8LHMS0Rk\nyokeos3MfgU4CTgGsOxyd99QgXqJiKRK7BM87cA/AzOB2cBzQBPhcPyEitVORCQlYi/wfBz4qLvP\nAXZl/rwa+HTFaiYikiKxYXki8Mm8ZdcBf1ne6oiIpFNsWI4SDr8BnjSzBcDRhInHRETqXmxY3gFc\nkPn7GmAI2Ey411JEpO7F3jq0LOfvqzJDsjUB36hUxURE0mTSsDSzacCDwAJ33wvg7sOVrpiISJpM\nehju7vuB/YQxLUVEpqTYm9I/AXzBzK4FtgGe/cDdH65ExURE0iQ2LK/P/Pm7ecsdmFa+6oiIpFPs\nBZ6CJjYTEak3CkERkQixz4b/FznnKXO5+1llrZGISArFnrPMH+T3WKALWFfe6oiIpFPsOctb85eZ\n2e3ALcCHy10pEZG0KeWc5Y+BXy9XRURE0iz2nOWf5C16EfAHwHfKXiMRkRSKPWf5R3nvdwPfJoxz\nKSJS92LPWXZUuiIiImkWdc7SzN5uZr+et+wUM8vvcYqI1KXYCzxXE+bbyfU4cE15qyMikk6xYZmd\npCzXKHBUeasjIpJOsWF5H3BJ3rLfB0Zid2Rm68zsSTN7zsweNLN3xJYVEam22Kvhfw18zczeCjwE\nvBI4m19MNRFjJdDl7nvN7GRgo5n9j7tvLqjGIiJVENWzzIyMvhD4HnAkcDew0N2/Fbsjd783O9I6\n4TlzB15RWHVFRKrD3A86PsbElcxmAgfcfTxn2QygIScAY7bzaWAxcATwP8BZ7j6Wt85SYClAc3Nz\n2+DgYNS2x8bGaGysj8km1ZZ0UlvSqZi2dHR0bHb30wsq5O6TvoD/BH4zb9lvAhtjyueVmwa0Ax8E\nZhxu3ba2No81NDQUvW7aqS3ppLakUzFtATZ5gdkVe4Hn1cB385bdDZxSUDKHcN7v4bD+OODdhZYX\nEamG2LAcBZrzljUTHnss1nR0zlJEakRsWN4O9JvZQjN7kZm9Gvg88M8xhc3sV83sUjNrNLNpZnYu\n0An8e3HVFhFJVmxY9hDuqbwbGCOMNjRCOO8YwwmH3NuAZ4BVwDJ3v7Og2oqIVEnsQBp7gD83syuA\nY4BfA94O/BB4WUT5nwKvL6GeIiJVFT34r5m9FPgL4BuE235OB95ToXqJiKTKYXuWmXspLybcG3ku\n8CNgAJgHvMXdf1Lh+omIpMJkPcungBuBBwj3WS5w96uB6BvRRUTqwWRh+X+EkYVeC5xhZkdXvkoi\nIulz2LB090WEeyG/CVwFbDezuwjPh8+oeO1ERFJi0gs87r7V3a9291cRRhp6EjgA3GNmH610BUVE\n0qCgqXDdfdjdlwLHAt2ExyBFROpeUfOGu/sedx9w9/PLXSERkTQqKixFRKYahaWI1KSBgQEWLlzI\n5s2bWbhwIQMDAxXdX+y0EiIiqTEwMEBPTw99fX0AnHbaaXR1dQHQ2dlZkX2qZykiNae3t5e+vj46\nOjowMzo6Oujr66O3t7di+1RYikjNGRkZob29fcKy9vZ2RkaiJ5wtmMJSRGpOS0sLw8PDE5YNDw/T\n0tJSsX0qLEWk5vT09NDV1cXQ0BDuztDQEF1dXfT09FRsn7rAIyI1J3sRp7u7myVLlnDLLbfQ29tb\nsYs7oJ6liNSozs5OtmzZQltbG1u2bKloUILCUkQkisJSRCSCwlJEJILCUkQkgsJSRCSCwlJEJILC\nUkTKLjsi0LRp0xIZESgJuildRMoqd0Sg9vZ2hoeHKz4iUBLUsxSRssodEWjGjBmJjAiUBIWliJRV\nNUYESoLCUkTKqhojAiVBYSkiZZU7ItD4+HgiIwIlQRd4RKSsckcEGhkZoaWlpeIjAiVBPUuRKSSp\nSb6yIwLt378/kRGBkqCepcgUUY1JvuqJepYiU0Q1JvmqJwpLkSmiXm/pSYrCUmSKqNdbepKSSFia\n2Uwz6zOzrWa2y8z+18zOT2LfIhJUY5KvepLUBZ7pwOPA64HHgAuAL5jZq9390YTqIDKlVWOSr3qS\nSM/S3Xe7+3J3f9TdD7j7V4BHgLYk9i8iQdKTfNUTc/fkd2rWDGwFTnX3+/M+WwosBWhubm4bHByM\n2ubY2BiNjY3lrmpVqC3ppLakUzFt6ejo2OzupxdUyN0TfQEzgH8Dbpxs3ba2No81NDQUvW7aqS3p\npLakUzFtATZ5gdmV6NVwM2sAPg/sA64o57Z37txZd4ONikh6JPYEj5kZ0Ac0Axe4+3i5tj0wMMCO\nHTtYvXp1XQ02KiLpkWTP8jNAC3CRu79Qzg339vYyd+7cuhtsVKaWepyKoZ4k0rM0s7nAO4G9wPbQ\nyQTgne5+W6nbHxkZ+aUTvHoyQWpJvU7FUE+SunVoq7ubu89y98acV8lBCeHJhLGxsQnL9GSC1JJ6\nnYqhntTF4449PT1s3bq17gYblalDz22nX12EZWdnJy972cvo7u5m1qxZdHd368kEqSl6bjv96mY8\nyzlz5rBly5ZqV0OkKNnntvPPWeowPD3qJixFalm9TsVQTxSWIinR2dmpcEyxujhnmaX71ESkUuqm\nZ7lz507dpyYiFVM3Pcvt27frPjURqZi6CcsXXnhB96mJSMXUTVgeccQRuk9NKiKpubYl3ermnOWx\nxx5LV1cXl112GevXr2dkZITZs2dz+eWXV7tqUsM017Zk1U3Pcs6cObzhDW9gzZo1rF69mj179nDH\nHXfw1a9+VT0BKZrm2pasuglLgKGhIW677Ta2b9/Oa17zGs455xwaGhp43/veV+2qSY3SM9uSVVdh\nOTIywrZt2+jp6fl57/Lmm29mfHxcvUspip7Zlqy6CsuWlhZWrFjxS7cQDQwM6LBJiqK5tiWrrsKy\np6eHhx9++OeHTQMDAxx//PFcfPHF3HfffcyfP189TClIZ2cnvb29dHd38/3vf18jWk1hdRWWnZ2d\nzJ07l+HhYQYGBli2bBkNDQ3ceeed7N27lzVr1vD+979fgSkF0VzbAnV061DWtddeS1dXFw0NDTQ1\nNXHzzTfT0dEBQEdHB2vXrqW7u1v/4UWkIHXVs4RfHDY9/PDDPPLII7S3t//8cLypqYmzzz6brVu3\nqncpIgWpu7CEEJgLFixg/vz5rFixgmXLljE6Osr4+DhNTU3s3r2bd73rXZx33nnVrqqUSCNNSVLq\nMiwhXOzZtWsXN9xwA3v37uXAgQMcffTRrF+/nr1797J+/Xruueceuru7q11VKVL26ZrsbWKrV6+m\np6dHgSkVUbdh2dnZySc+8QlGR0dxd2bMmEF/fz/bt2/nhBNO4OKLL+app57i1ltv1ZerRmlGRElS\n3YYl/OJwfNeuXYyOjrJt27ZfukL+5S9/WVfIKyCJwSf0dI0kqa7DEsLheFNTEy9+8YtZsWIFTU1N\nrF27dkJvZO3ateqNlFHu4fFpp51WscNjPV0jSar7sOzs7OTMM89kfHx8whXyXOqNlFdSg0/kPl2j\n+eKl0uo+LAG+/vWvs2TJEhobG5k/f35RvRFddY2X1OFx7tM1mi9eKm1KhCXA6tWrufHGG9m1axeL\nFy+e0BtZvHjxYXsj9XbVtdLBn+Thcfbpmv379+vpGqksd0/tq62tzWMNDQ1Frdff3+/HHXecNzY2\nupn5vHnzvL+//7BlWltbfcOGDROWbdiwwVtbW6PrV4jYthSjv7/f58+f7xs2bPB9+/b5hg0bfP78\n+ZP+GxS7j+yr3Puohkr+XJI21dsCbPIC86jqgXi4VyXCshgNDQ2+b9++Ccv27dvnDQ0NZd1Pf3+/\nt7a2+qpVq7y1tbUi4ZJU8CfRlqRN9YBJq6TCsu6eDa+E7GFl9hlzKP9hZVLTFyR5PrGzs5ONGzdy\n5ZVXlnXbItUwZc5ZliKJq65JXUHW7TYixVHPMkK2Z9fd3c3IyAgtLS1lv+qaVI8vG/x9fX20t7cz\nPDxMV1eX7jMVmYTCMlL2sLJSkjjUh2SCX6QeKSxTIrfHB/z8UL8SPb5KB79IPVJYpkRuj2/JkiXc\ncsst6vGJpEhiF3jM7Aoz22Rme81sbVL7rSWavkAkvZLsWT4BXAOcCxyR4H5FREqWWFi6+x0AZnY6\ncFxS+xURKQcLN7MnuEOza4Dj3H3xIT5fCiwFaG5ubhscHIza7tjYGI2NjeWqZlWpLemktqRTMW3p\n6OjY7O6nF1So0Ed+Sn0RDsXXxqyblscdk6a2pJPakk5JPe6oJ3hERCIoLEVEIiR2gcfMpmf2Nw2Y\nZmazgJ+5+88OVWbz5s1Pm9nWyF0cAzxdek1TQW1JJ7UlnYppy9xCd5LYBR4zWw78Xd7iFe6+vEzb\n3+SFnrBNKbUlndSWdEqqLUneOrQcWJ7U/kREyknnLEVEItRTWN5U7QqUkdqSTmpLOiXSlsRvShcR\nqUX11LMUEakYhaWISISaD0szm2NmXzKz3Wa21cwuq3J9DjkUnZmdbWb3m9nzZjZkZnNzPptpZmvM\n7Dkz225m7y1X2SLbMdPM+jL/prvM7H/N7PxabEtmu+vM7MnMdh80s3fUaltytv8qM9tjZutyll2W\n+ZntNrP1ZjYn57PDfldKKVtCGzZm2jCWeT2Q2rYU+nxk2l7AAPBPQCPQDowCrVWszx8AbwI+Q84z\n8IQbZ0eBPwRmAR8DvpPz+Urgv4CjgRZgO3BeqWVLaMeRhFu95hF+qV4I7Mq8r6m2ZLbbCszM/P3k\nzHbbarEtOdv/Zmb763LauAs4K/N96AcGY74rpZQtsQ0bgXcc4ueVqrZUJVDK+J/lSGAfcGLOss8D\n16WgbhMGDCGMpPTtvLq/AJycef8E8Hs5n1+d/QGXUrbMbfo/4JJabwtwEvAk8JZabQtwKfAFwi+0\nbFheC/TnrPOKzPejabLvSillS2zHRg4elqlrS60fhp9IeGTywZxl9xB+s6RNK6FuALj7buAhoNXM\njgZ+LfdzJrajlLJlYWbNhH/ve2u1LWb2aTN7HrifEJZfq8W2mNls4MNA/mF9fn0eIhMMTP5dKaVs\nqVaa2dNm9i0zW5TWttR6WDYCz+UtGyX8BkmbRkLdcmXr2pjzPv+zUsuWzMxmALcBt7r7/SXWp2pt\ncfc/y2zrdcAdwN4S61OttlwN9Ln7trzlk9XncN+VUsqW4q+BE4CXE+6XvMvMXlFifSrSlloPyzFg\ndt6y2YTzFWlzuLqO5bzP/6zUsiUxswbCYco+4Ioy1KdqbQFw9/3uPkwYrf/dJdYn8baY2anAOcDH\nD/LxZPU53HellLJFc/fvuvsud9/r7rcC3wIuKLE+FWlLrYflg8B0M3tVzrJTCIeKaXMvoW4AmNmR\nhHMp97r7M4TDwlNy1s9tRylli2ZmBvQBzcAl7j5eq205iOnZ/ZZQn2q0ZRHhIttjZrYduAq4xMy+\nf5D6nADMJHxPJvuulFK2nBywEutTmbaU88R5NV7AIOHq1pHAb1P9q+HTCVdGVxJ6ZLMyy16aqdsl\nmWUfYeKV0+uA/yBcOT2Z8EXLXnUtumyJbfks8B2gMW95TbUF+FXCBZFGwhCB5wK7gYtrsC0vAo7N\nea0CvpipSyvhEPN1me/DOiZeBT7kd6WUsiW05ajMzyL7HXlb5udyYhrbUpVAKecLmAOsz/wjPwZc\nVuX6LCf8dsx9Lc98dg7h4sILhKuA83LKzQTWZH7ITwHvzdtu0WWLbMfcTN33EA5dsq+31WBbXkoI\nrWcz2/0B8KflqE/SbTnE/7d1Oe8vy3wPdgNfBubEfldKKVvCz+V7hEPgZwm/mH83rW3Rs+EiIhFq\n/ZyliEgiFJYiIhEUliIiERSWIiIRFJYiIhEUliIiERSWkgqZsQxPKHEbnzWzD5WrTkltW2qD7rOU\ngpnZo4Rhtf4tYt2NhJumP1fpehUr055m4GfAfuA+4B+Bm9z9QBWrJiminqVUlZklNnf9JC5y9ybC\nk0vXEUbD6atulSRNFJZSEjNbbGbDZrbKzJ4xs0csM/2EmfUSns+9PnOYfX1muZvZn5vZD4Ef5ix7\nZebva83sBjP7qoUpLb6bGbYLCz5uZj+xMF3DD8xsYU65a3Lq9kYL02E8Z2YPmdl5k7XH3Ufd/U7g\nrcAfH2zbZrbIzLaZ2fsy9XjSzN5kZhdYmLJip5l9oGz/yJIKCksph9cCDxCmWfgo0Gdm5u49hGkP\nrnD3Rne/IqfMmzLlFhxim5cCKwiDUPwI6M0s/z3CdAEnAi8mjHa+I7+wmf0G4VD6rwgDNpwFPBrb\nIHe/G9hGCPuDOZYwAMTLgb8FbgYuJ0xV8TrgQ2Y2P3Z/kn4KSymHre5+s7vvB24ljBDePEmZle6+\n091fOMTnX3L3u939Z4SBh0/NLB8nDNR6MuGc+4i7P3mQ8l3AGnf/V3c/4O4/9jBwcSGeIAy6cDDj\nQK+HYesGCb8oPulhbMZ7Cec9TzlEWalBCksph+3Zv7j785m/Nh5i3azHY7cJPJ/dnrtvAK4HbgB+\nYmY3ZaZZyHc8YYqHUrwc2HmIz3ZkfjlAGHEIwshC5Cyb7N9AaojCUirtULdbFH0bhrt/yt3bCIfw\nJxIOtfM9ThiItyhmdgYhLIeL3YbUF4WlVNpThDlWysLMzjCz12bmBdpNGG/zYLf39AFLLMzr3WBm\nLzezkyO2P9vMLiQcWq9z9x+Uq+5S2xSWUmmfBN6cuVL+qTJsbzbhYsozwFbCxZ2P5a+UuUCzhDBX\nzShh8N+5h9nuXWa2i9Aj7QH+IVNeBNBN6SIiUdSzFBGJoLAUEYmgsBQRiaCwFBGJoLAUEYmgsBQR\niaCwFBGJoLAUEYmgsBQRifD/IepjcDPtyVgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2fc96d8290>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "nn = len(Rs)-1\n",
    "fig, ax = subplots(figsize=(5,4) )\n",
    "                  \n",
    "plt.scatter(dim[1:], Rs[1:,0], edgecolor=\"k\", facecolor=\"w\" )\n",
    "ax.plot(dim[1:], Rs[0,0]*np.ones(nn)/0.9,'r-.', label=\"Testing: baseline\")\n",
    "ax.set_xlabel('Intrinsic Dim')\n",
    "ax.set_ylabel('Accuracy')\n",
    "plt.grid()\n",
    "# ax.set_ylim([-0.1,1.1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above figure show that updating in the intrinsic space can prevent overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['../results/lrb_lenet_cifar/170818_014656_6c7951e_lrb_lenet_cifar_dir_0/diary', '../results/lrb_lenet_cifar/170820_212952_6c7951e_lrb_lenet_cifar_lrb_18000/diary', '../results/lrb_lenet_cifar/170820_213003_6c7951e_lrb_lenet_cifar_lrb_17000/diary', '../results/lrb_lenet_cifar/170820_205254_6c7951e_lrb_lenet_cifar_lrb_10/diary', '../results/lrb_lenet_cifar/170818_021257_6c7951e_lrb_lenet_cifar_lrb_100000/diary', '../results/lrb_lenet_cifar/170820_204628_6c7951e_lrb_lenet_cifar_lrb_1500/diary', '../results/lrb_lenet_cifar/170820_204627_6c7951e_lrb_lenet_cifar_lrb_2000/diary', '../results/lrb_lenet_cifar/170820_205619_6c7951e_lrb_lenet_cifar_lrb_100/diary', '../results/lrb_lenet_cifar/170820_214922_6c7951e_lrb_lenet_cifar_lrb_25000/diary', '../results/lrb_lenet_cifar/170820_204647_6c7951e_lrb_lenet_cifar_lrb_4000/diary', '../results/lrb_lenet_cifar/170820_205142_6c7951e_lrb_lenet_cifar_lrb_50/diary', '../results/lrb_lenet_cifar/170818_030211_6c7951e_lrb_lenet_cifar_lrb_150000/diary', '../results/lrb_lenet_cifar/170820_205118_6c7951e_lrb_lenet_cifar_lrb_1750/diary', '../results/lrb_lenet_cifar/170820_213052_6c7951e_lrb_lenet_cifar_lrb_19000/diary', '../results/lrb_lenet_cifar/170820_204555_6c7951e_lrb_lenet_cifar_lrb_250/diary', '../results/lrb_lenet_cifar/170820_204616_6c7951e_lrb_lenet_cifar_lrb_5000/diary', '../results/lrb_lenet_cifar/170820_204616_6c7951e_lrb_lenet_cifar_lrb_750/diary', '../results/lrb_lenet_cifar/170818_021211_6c7951e_lrb_lenet_cifar_lrb_21000/diary', '../results/lrb_lenet_cifar/170818_021258_6c7951e_lrb_lenet_cifar_lrb_50000/diary', '../results/lrb_lenet_cifar/170820_205035_6c7951e_lrb_lenet_cifar_lrb_1000/diary', '../results/lrb_lenet_cifar/170818_024836_6c7951e_lrb_lenet_cifar_lrb_40000/diary', '../results/lrb_lenet_cifar/170820_213108_6c7951e_lrb_lenet_cifar_lrb_20000/diary', '../results/lrb_lenet_cifar/170820_213036_6c7951e_lrb_lenet_cifar_lrb_16000/diary', '../results/lrb_lenet_cifar/170820_205101_6c7951e_lrb_lenet_cifar_lrb_500/diary', '../results/lrb_lenet_cifar/170820_204603_6c7951e_lrb_lenet_cifar_lrb_1250/diary', '../results/lrb_lenet_cifar/170820_213130_6c7951e_lrb_lenet_cifar_lrb_45000/diary', '../results/lrb_lenet_cifar/170818_021311_6c7951e_lrb_lenet_cifar_lrb_30000/diary', '../results/lrb_lenet_cifar/170820_205131_6c7951e_lrb_lenet_cifar_lrb_3000/diary', '../results/lrb_lenet_cifar/170820_204547_6c7951e_lrb_lenet_cifar_lrb_10000/diary', '../results/lrb_lenet_cifar/170820_205141_6c7951e_lrb_lenet_cifar_lrb_15000/diary', '../results/lrb_lenet_cifar/170820_213029_6c7951e_lrb_lenet_cifar_lrb_35000/diary', '../results/lrb_lenet_cifar/170820_205019_6c7951e_lrb_lenet_cifar_lrb_2500/diary']\n",
      "[0, 10, 50, 100, 250, 500, 750, 1000, 1250, 1500, 1750, 2000, 2500, 3000, 4000, 5000, 10000, 15000, 16000, 17000, 18000, 19000, 20000, 21000, 25000, 30000, 35000, 40000, 45000, 50000, 100000, 150000]\n",
      "(32, 5)\n",
      "Intrinsic dim is: 40000 for LeNet\n"
     ]
    }
   ],
   "source": [
    "results_dir = '../results/lrb_lenet_cifar' \n",
    "i = 0        \n",
    "\n",
    "# filename list of diary\n",
    "diary_names = []\n",
    "for subdir, dirs, files in os.walk(results_dir):\n",
    "    for file in files:\n",
    "        if file == 'diary':\n",
    "            fname = os.path.join(subdir, file)\n",
    "            diary_names.append(fname)\n",
    "\n",
    "print diary_names            \n",
    "dim_ = []            \n",
    "for f in diary_names: \n",
    "    dim_.append( int(f.split('/')[-2].split('_')[-1]) )         \n",
    "    \n",
    "dim = sorted(dim_) \n",
    "\n",
    "print dim\n",
    "            \n",
    "diary_names_ordered = []\n",
    "for d in dim:\n",
    "    for f in diary_names:\n",
    "        if d ==  int(f.split('/')[-2].split('_')[-1]):\n",
    "            # print \"%d is in\" % d + f\n",
    "            diary_names_ordered.append(f)        \n",
    "\n",
    "# intrinsic update method\n",
    "Rs, Acc = [], []\n",
    "i = 0\n",
    "for fname in diary_names_ordered:\n",
    "    with open(fname,'r') as ff:\n",
    "        lines0 = ff.readlines()\n",
    "        R = extract_num(lines0)\n",
    "        # print \"%d dim:\\n\"%dim[i] + str(R) + \"\\n\"\n",
    "        i += 1\n",
    "\n",
    "        Rs.append(R)\n",
    "        Acc.append(R[1])\n",
    "                            \n",
    "Rs = np.array(Rs)\n",
    "Acc = np.array(Acc)\n",
    "\n",
    "print Rs.shape\n",
    "\n",
    "# 2.2 construct acc_solved_all and dim_solved_all           \n",
    "\n",
    "for id_d in range(len(dim)):\n",
    "    d = dim[id_d]\n",
    "\n",
    "    r = Rs[id_d,1]\n",
    "    if d==0:\n",
    "        test_acc_bl = r        \n",
    "        # print \"Acc goal is: \" + str(test_acc_sl) + \" for network with depth \" + str(ll) + \" width \"+ str(w)\n",
    "    else:\n",
    "        test_acc = r\n",
    "        if test_acc>test_acc_bl*0.9:\n",
    "            acc_solved=test_acc\n",
    "            dim_solved=d\n",
    "            print \"Intrinsic dim is: \" + str(d) + \" for LeNet\"\n",
    "            # print \"\\n\"\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0.6469],[10, 0.1176],[50, 0.1026],[100, 0.1049],[250, 0.1126],[500, 0.1674],[750, 0.2198],[1000, 0.2655],[1250, 0.2956],[1500, 0.3113],[1750, 0.3207],[2000, 0.3613],[2500, 0.333],[3000, 0.3506],[4000, 0.3959],[5000, 0.4146],[10000, 0.5138],[15000, 0.5377],[16000, 0.5384],[17000, 0.5239],[18000, 0.5364],[19000, 0.5355],[20000, 0.5514],[21000, 0.5423],[25000, 0.5521],[30000, 0.5752],[35000, 0.5747],[40000, 0.5881],[45000, 0.6003],[50000, 0.6058],[100000, 0.5749],[150000, 0.1627]\n"
     ]
    }
   ],
   "source": [
    "print ','.join(['[%i, %s]' % (dim[n], Acc[n]) for n in xrange(len(Acc))])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\" Extract final stats from resman's diary file\"\"\"\n",
    "def extract_num(lines0, is_reg=False):\n",
    "\n",
    "    if is_reg:\n",
    "        valid_loss_str     = lines0[-5]\n",
    "        valid_accuracy_str = lines0[-6]\n",
    "        train_loss_str     = lines0[-8]\n",
    "        train_accuracy_str = lines0[-9]\n",
    "        average_time_str   = lines0[-10]        \n",
    "        run_time_str       = lines0[-11]   \n",
    "        \n",
    "    else: \n",
    "        valid_loss_str     = lines0[-6]\n",
    "        valid_accuracy_str = lines0[-7]\n",
    "        train_loss_str     = lines0[-10]\n",
    "        train_accuracy_str = lines0[-11]\n",
    "        average_time_str   = lines0[-12]        \n",
    "        run_time_str       = lines0[-13]\n",
    "\n",
    "    valid_loss     = float(valid_loss_str.split( )[-1])\n",
    "    valid_accuracy = float(valid_accuracy_str.split( )[-1])\n",
    "    train_loss     = float(train_loss_str.split( )[-1])\n",
    "    train_accuracy = float(train_accuracy_str.split( )[-1])\n",
    "    run_time       = float(run_time_str.split( )[-1])\n",
    "    \n",
    "    return valid_loss, valid_accuracy, train_loss, train_accuracy, run_time\n",
    "\n",
    "\"\"\" Extract number of total parameters for each net config from resman's diary file\"\"\"\n",
    "def parse_num_params(lines0):\n",
    "    line_str = ''.join(lines0)\n",
    "    idx = line_str.find(\"Total params\")\n",
    "    param_str = line_str[idx+14:idx+14+20] # 14 is the length of string \"Total params: \"\n",
    "    param_num = param_str.split(\"\\n\")[0]\n",
    "    return int(locale.atof(param_num))\n",
    "\n",
    "def extract_perf_dim(results_dir):\n",
    "    # Dim: subspace dim\n",
    "    # Acc: Accuracy\n",
    "    \n",
    "    # filename list of diary\n",
    "    diary_names = []\n",
    "    for subdir, dirs, files in os.walk(results_dir):\n",
    "        for f in files:\n",
    "            if f == 'diary':\n",
    "                fname = os.path.join(subdir, f)\n",
    "                diary_names.append(fname)\n",
    "\n",
    "    # print diary_names   \n",
    "\n",
    "    dim = []\n",
    "    for f in diary_names:\n",
    "        # print f\n",
    "        tmp_str = f.split('/')[-2]\n",
    "        if tmp_str.split('_')[-3]=='LeNet':\n",
    "            d = int(tmp_str.split('_')[-2])\n",
    "            dim.append(d)\n",
    "\n",
    "    dim = list(set(dim))\n",
    "    dim = sorted(dim)  \n",
    "\n",
    "    # print dim\n",
    "\n",
    "    diary_names_ordered = []\n",
    "    for d in dim:\n",
    "        for f in diary_names:\n",
    "            if '_LeNet_'+str(d)+'_' in f and f.split('_')[-3]=='LeNet':\n",
    "                # print \"%d is in\" % d + f\n",
    "                diary_names_ordered.append(f)           \n",
    "    # print diary_names_ordered\n",
    "    # intrinsic update method\n",
    "    Dim= []\n",
    "    Acc = []\n",
    "\n",
    "    for fname in diary_names_ordered:\n",
    "        tmp_str = fname.split('/')[-2]\n",
    "        d = int(tmp_str.split('_')[-2])\n",
    "        with open(fname,'r') as ff:\n",
    "            lines0 = ff.readlines()\n",
    "\n",
    "            try: \n",
    "                r = extract_num(lines0,False)[1]\n",
    "                # print \"%d dim:\\n\"% d + str(r) + \"\\n\"\n",
    "                Dim.append(d)\n",
    "                Acc.append(r)\n",
    "\n",
    "            except ValueError:\n",
    "                print \"%d dim:\\n\"%d + \"Error \\n\"\n",
    "                pass\n",
    "\n",
    "    return Dim, Acc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_perf_dim(Dim, Acc, Cx=None):\n",
    "    fig, ax = subplots(figsize=(6,5) )\n",
    "\n",
    "    font = {'size'   : 12}\n",
    "    matplotlib.rc('font', **font)\n",
    "\n",
    "    for i in range(len(Dim)):\n",
    "        if Cx == None:\n",
    "            if Acc[i]>Acc[0]*0.9 and i>10:\n",
    "                print \"d_{int}=\"+str(Dim[i]) + ', acc:' + str(Acc[i])\n",
    "                break\n",
    "        else: \n",
    "            if Acc[i]>Cx and i>10:\n",
    "                print \"d_{int}=\"+str(Dim[i]) + ', acc:' + str(Acc[i])\n",
    "                break\n",
    "                \n",
    "            \n",
    "    \n",
    "    plot(Dim[4:], Acc[4:], 'o', mec='b', mfc=(.8,.8,1), ms=10)\n",
    "    plot(Dim[i], Acc[i], 'o', mec='b', mfc='b', ms=10)\n",
    "    axhline(Acc[0], ls='-', color='k',label='baseline')\n",
    "    axhline(Acc[0] * .9, ls=':', color='k',label='solved')\n",
    "    plt.legend()\n",
    "    ax.set_xlabel('Subspace dimension $d$')\n",
    "    ax.set_ylabel('Validation accuracy')\n",
    "\n",
    "    # ax.set_title('Cifar: Untied_LeNet' )\n",
    "    plt.grid()\n",
    "    ax.set_ylim([0.0,1.01])\n",
    "    # fig.savefig(\"figs/fnn_mnistPL_W\"+str(width[i])+\"_L\"+str(depth[j])+\".pdf\", bbox_inches='tight')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "d_{int}=1750, acc:0.5015\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAFKCAYAAADsVHOaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XmYFNXZ9/Hvzb45CLILgmJiXOIGLk8Mj20k4hY3QBEw\n0eQBjRjAiHEJMVFR0cSIcYlLQoiggiCIRlGjgG8So4KKRDQok4gssgkq+3q/f5yaoRlmumt6pnt6\nZn6f66qru6pOVZ3TXV13n1rOMXdHREQklTpVnQEREcl/ChYiIpKWgoWIiKSlYCEiImkpWIiISFoK\nFiIikpaChYiIpKVgISIiaeUsWJjZVWY218y2mtm4NGmvNrMVZvaVmY01s4Y5yqaIiJTCcvUEt5ld\nAOwCegGN3f3SMtL1Ah4DvgMsB6YBb7j79anW36pVK+/SpUtGedu4cSNNmzbNaNl8oTLkh5pQBqgZ\n5VAZ4nn77bfXuHvrtAndPacDMAoYl2L+E8DtSeOnAivSrbdbt26eqVmzZmW8bL5QGfJDTSiDe80o\nh8oQDzDXYxy78/GaxeHAe0nj7wFtzWy/KsqPiEitl7PTUMUbNBsFdPSyT0MVAkPc/cVovD6wDTjQ\n3T8pkXYwMBigbdu23SZOnJhRnjZs2ECzZs0yWjZfqAz5oSaUAWpGOVSGeE455ZS33b17unT1spqL\nzGwACpLGi96vL5nQ3R8BHgHo3r27JxKJjDY4e/ZsMl02X6gM+aEmlAFqRjlUhsqVj6ehFgBHJY0f\nBax098+rKD8iIrVezmoWZlYv2l5doK6ZNQJ2uPuOEkkfA8aZ2eOEu6FGAuNylU8Rya5du3axZs0a\nvvjiC3bu3Jm17TRv3pwPP/wwa+vPhcoqQ6NGjejYsSP169fPeB25PA01Evhl0vhA4GYzGwt8ABzm\n7p+6+4tmdhcwC2gMPF1iORGpxpYuXYqZ0aVLF+rXr4+ZZWU769evZ5999snKunOlMsrg7nz++ecs\nXbqUAw88MOP15CxYuPuvgF+VMXuPKzju/lvgt1nOkohUgY0bN3LIIYdQp04+ngWvecyM/fbbj9Wr\nV1doPfq2RCTnFChyqzJqb/rGREQkLQULEZEkXbp04ZVXXsnZ9syMRYsWAXDFFVdw66235mzb5ZGP\nz1mIiNRKDz30UFVnoUyqWYiISFoKFiIiJcyZM4fDDjuMFi1acNlll7FlyxbWrVvH2WefTevWrWnR\nogVnn302S5cuLV5m3LhxHHTQQeyzzz4ceOCBPP7448Xzxo4dy6GHHkqLFi3o1asXixcvLnW7l156\nKSNHjgTC09vf+MY3uPvuu2nTpg3t27fnT3/6U3HarVu3MmLECA444ADatm3LFVdcwebNm7P0iShY\niIjs5fHHH+ell16isLCQjz76iFGjRrFr1y4uu+wyFi9ezKeffkrjxo256qqrgHA78NChQ5kxYwbr\n16/n9ddf5+ijjwZg+vTp3H777UydOpXVq1fTo0cPLr744lj5WLlyJV9++SXLli3jj3/8I0OGDGHd\nunUAXH/99Xz00UfMmzePRYsWsWzZMm655ZbsfCDomoWIVLHhw4czb968Sl/vzp07qVu3LgBHH300\nY8aMib3sVVddRadOnQD4+c9/zk9+8hNGjRpF7969i9P8/Oc/55RTTiker1OnDu+//z4HHHAA7du3\np3379kC4DnHDDTdw6KGHAnDjjTdy++23s3jxYjp37pwyH/Xr1+emm26iXr16nHnmmTRr1oyFCxdy\nwgkn8MgjjzB//nxatmxZvN7+/ftzxx13xC5neahmISJSQlGgAOjcuTPLly9n06ZNXH755XTu3JmC\nggL+93//t7jJkqZNmzJp0iQeeugh2rdvz1lnncW///1vABYvXsywYcPYd9992XfffWnZsiXuzrJl\ny9Lmo2XLltSrt/s/fZMmTdiwYQOrV69m06ZNdOvWrXi9p59+eoUfvEtFNQsRqVLl+cdfHhVpKmPJ\nkiXF7z/99FM6dOjA3XffzcKFC3nzzTdp164d8+bN45hjjinqpI1evXrRq1cvNm/ezMiRIxk0aBB/\n+9vf6NSpEz//+c8ZMGBApZQLoFWrVjRu3JgFCxaw//77V9p6U1HNQkSkhAceeIClS5eydu1abrvt\nNi666CLWr19P48aN2XfffVm7di0333xzcfqVK1cyffp0Nm7cSMOGDWnWrFnxU+pXXHEFd9xxBwsW\nLADgyy+/ZPLkyRXKX506dRg0aBBXX301q1atAmDZsmW89NJLFVpvym1mbc0iItVU//79Oe200zjo\noIPo2rUrI0eOZPjw4WzevJlWrVpx4okncvrppxen37VrF7/97W/p0KEDLVu25LXXXuP3v/89AOef\nfz7XXXcd/fr1o6CggCOOOIIZM2ZUOI933nknBx98MCeeeCIFBQX07NmThQsXVni9Zcl5T3nZ0r17\nd587d25Gy+ZTByOZUhnyQ00oA2S3HB9++GHxxd5sUquzeyrrczezWD3lqWYhIiJpKViIiEhaChYi\nIpKWgoWIiKSlYCEiImkpWIiISFoKFiIikpaChYiIpKVgISJSAbNnz6Zjx46Vvt5PPvmEgoICduzY\nUenrzoSChYiIpKVgISIiaSlYiIgkufPOO9l///3ZZ599OOSQQ3j11VfZunUrw4cPp0OHDnTo0IHh\nw4ezdevWUpft06fPHtOGDRvG0KFDgdDi7I9+9CPat2/P/vvvz8iRI9m5cycQOmsaMWIErVq14qCD\nDuL555/PfmHLQcFCRCSycOFC7r//fubMmcP69et56aWX6NKlC7fddhtvvPEG8+bN47333uOtt95i\n1KhRey3fr18/XnjhBdavXw+EAPDUU0/Rv39/IPSxXa9ePRYtWsS7777Lyy+/zB/+8AcAHn30Uf7y\nl7/w7rvvMnfuXKZMmZK7gsegYCEiVS6RSDBu3DgAtm/fTiKRYMKECQBs2rSJRCLBpEmTgPDvPJFI\nMHXqVADWrFlDIpHgueeeA2DFihUkEgn++te/Ant2ZJRO3bp12bp1Kx988AHbt2+nS5cudO3alccf\nf5ybbrqJNm3a0Lp1a375y18yfvz4vZbv3Lkzxx57LNOmTQNg5syZNGnShBNPPJGVK1fywgsvMGbM\nGJo2bUqbNm24+uqrmThxIgBPPfUUw4cPp1OnTrRs2ZIbbrghg08yexQsREQiBx98MGPGjOFXv/oV\nbdq0oV+/fixfvpzly5fv0V92UVerpenfvz9PPvkkAE888URxrWLx4sVs376d9u3bF3eFevnllxd3\nXrR8+fK9unPNJ+pWVUSq3OzZs4vf169ff4/xJk2a7DHevHnzPcZbtWq1x3i7du2YPXt28amg5ANw\nHP3796d///589dVXXH755Vx33XV06NCBxYsXc/jhhwO7u1otTd++fbnmmmtYunQp06ZN45///Gdx\nPho2bMiaNWv26Fe7SPv27ffqzjWfqGYhIhJZuHAhM2fOZOvWrTRq1IjGjRtTp04dLr74YkaNGsXq\n1atZs2YNt9xyCwMHDix1Ha1btyaRSHDZZZdx4IEHFnc41L59e0477TSuueYavvrqK3bt2kVhYSGv\nvfYaABdeeCG/+93vWLp0KevWrWP06NE5K3ccChYiIpGtW7dy/fXX06pVK9q1a8eqVau44447GDly\nJN27d+fII4/km9/8JsceeywjR44scz39+/fnlVdeKT4FVeSxxx5j27ZtHHbYYbRo0YI+ffrw2Wef\nATBo0CB69erFUUcdxbHHHssFF1yQ1bKWl7pVpWZ0haky5IeaUAZQt6r5Qt2qiohItaJgISIiaSlY\niIhIWgoWIiKSloKFiORcTbmxprqojM9bwUJEcqp+/fps3ry5qrNRq2zfvr3UBwHLQ8FCRHKqTZs2\nLFu2jE2bNqmGkQO7du1i5cqVNG/evELrUXMfIpJTBQUFQGgLafv27VnbzpYtW2jUqFHW1p8LlVWG\npk2b0qpVqwqtQ8FCRHKuoKCgOGhky+zZsznmmGOyuo1sy6cy6DSUiIikpWAhIiJpKViIiEhaOQsW\nZtbSzKaZ2UYzW2xm/ctI19DMHjKzlWa21syeM7P9c5VPERHZWy5rFg8A24C2wADg92Z2eCnphgH/\nAxwJdADWAfflKpMiIrK3nAQLM2sK9AZ+4e4b3P3vwLPAJaUkPxB4yd1XuvsWYBJQWlAREZEcyVXN\n4uvADnf/KGnae5QeBP4InGRmHcysCaEWMiMHeRQRkTLkpPMjM+sBTHb3dknTBgED3D1RIm1z4GHg\nImAn8C/gVHdfW8p6BwODAdq2bdtt4sSJGeVvw4YNNGvWLKNl84XKkB9qQhmgZpRDZYjnlFNOidX5\nEe6e9QE4BthUYto1wHOlpJ0ATANaAg2BXwBvpttGt27dPFOzZs3KeNl8oTLkh5pQBveaUQ6VIR5g\nrsc4jufqNNRHQD0z+1rStKOABaWkPRoY5+5r3X0r4eL28WZWsWfVRUQkYzkJFu6+EZgK3GJmTc3s\nJOBcYHwpyecA3zez5mZWH7gSWO7ua3KRVxER2Vsub529EmgMrAKeBH7s7gvMrIeZbUhKNwLYAnwM\nrAbOBM7PYT5FRKSEnDUk6OEC9XmlTP8b0Cxp/HPCHVAiIpIn1NyHiIikpWAhIiJpKViIiEhaChYi\nIpKWgoWIiKSlYCEiImkpWIiISFoKFiIikpaChYiIpKVgISIiaSlYiIhIWgoWIiKSloKFiIikpWAh\nIiJpKViIiEhasYKFmR2V7YyIiEj+iluzeMXM3jOzEWbWPqs5EhGRvBM3WLQHbgJOAD42s5fNbKCZ\nNcle1kREJF/EChbuvsPdp7t7X2B/4CngZ8BKM3vMzE7KZiZFRKRqlesCt5k1I/Sj3Q/oCEwEPgYe\nN7MHKj97IiKSD+rFSWRmZwGXAGcA/wD+ADzj7lui+Q8AnwJDspRPERGpQrGCBTAaeAy42t0/KznT\n3dea2fBKzZmIiOSNWMHC3b8ZI80fKp4dERHJR3Gfs5hqZj1KTOthZlOyky0REckncS9wnwy8XmLa\nP4FTKjc7IiKSj+IGiy1A0xLTmgHbKzc7IiKSj+IGi5eAh82sACB6vR94MVsZExGR/BE3WFwDFABr\nzWwVsBZoDugOKBGRWiDu3VDrgLOidqE6AkvcfUVWcyYiInkj7nMWALj7Z2a2AjAzqxNN25WVnImI\nSN6Ie+tsBzObZmafAzsIF7aLBhERqeHiXrN4GNgGnApsAI4FngWuyFK+REQkj8Q9DfUt4AB332hm\n7u7vmdmPCM9ePJq97ImISD6IW7PYSTj9BPCFmbUGNhKaKxcRkRoubrB4Ezgzev8SMAmYCszNRqZE\nRCS/xD0NdQm7A8twwnMX+wBjspEpERHJL2mDhZnVBe4FBgO4+2ZgVJbzJSIieSTtaSh33wmcBuh5\nChGRWiruNYt7gJvNrH42MyMiIvkpbrD4CXAtsN7MlpjZp0VDFvMmeaCwEIYNgzZtoG7d8DpsWJgu\nIrVH3GAxEOgJ9IreX5I0SA01YwZ06wazZsH26Fn97dvDeLduYX5FJQejU089OSvBSAFPpOJiBQt3\nf62sIdsZlKpRWAgXXgju0L07jBsHr78eXrt3D9MvvLBiB9wZM+D442H9enj0UXj9dePRR8P48cdX\nTjDaextU+jYkf+Tiz0dtFbdtqFvKGrKdQakat0Tf7JgxMGQIdOwI9eqF1yFDwnSAW2/NbP2FhTBw\nIPzsZ7BlC/zwh/Ctb4XXLVvC9IEDK/Yjz8U2ciUXtaPqfqCdMQOOOw6++GLPPx9ffBGm649BxcQ9\nDdWpxHAcMALomqV8SQwTJkCHDtCoEXznOyfTqFEYnzCh4uueNg369IEjjyx9/pFHQu/eMHVqZuv/\n3e/Cqaw77oD69WHs2PCvf+zYMH7HHWH+ffdlXoaibdx2G7z/PuyK7ufbtSuM33ZbxbcB2T/I5uJ0\nYC5qedlUWAgXXwx33w1Dh+7552bo0DD94ourR+CLuz/l+vRq3NNQl5UYzgAuYHcTIGmZWcuo5dqN\nZrbYzPqnSHusmf0/M9tgZivNbFjc7dQWl1wCgwbBd78LkybBP/9pTJoUxgcNCvMrYts2OO+81GnO\nPz+ky8SECfDGG3DPPaX/uO+5J8wfPz6z9Rdt4x//ALPST6WZhfkV2caMGWFd69bteZBdty5Mr+hB\nNhenA2tCDWzUKPje91L/uTn77PAHoSJmzgwBunFjqFMnvHbrFqZXhhkz4Nhjw/rCHwNj+/Ywfuyx\nu/enXPyBKCluzaI0LwNpDid7eIDQcm1bYADwezM7vGQiM2tF6K71YWA/4OBoWxKZMAGmTIEHHyz9\nQPvgg2F+RWoY27ZBu3ap07Rrl3mwWL8ezj039Y/7nHNCukx99VX4Qac6lVanTubbKCyEiy6C3/42\n/KNLXv+wYWH6RRdV7CBbdDrwxhtLP5DfeGOYn+npQAg1sGOPhdGjQy01uZbXqFGYfswxFa+BJdeE\n69ShUmvCTz8dasKp9O0bfheZuuUWOOssOOIImDgR/vnP8HrEEWH6LRU8KV9YuLsMxx235x+D444L\n0/v0CYHjwgtDDfmYY/ZMd8wxYXpF/0CUxtw9fSKzg0pMagL0B85x9yNiLN8UWAcc4e4fRdPGA8vc\n/foSaW8HOrl7uf4bd+/e3efOzaypqtmzZ5NIJDJatip06BBqEEOHlp3m3nvh1Vdh2bLMttGoUaix\ndOxYdpqlS6FfP9i8Of/WD+FfX79+cNVVZae5777wg9+ypfzr79MnbGN4is6F77knrDvTg1RBAZxw\nAsydG2py554bgvSKFTB9ejhd2K0bvPVWCI6ZaNEi1FDuuaf04D1/Plx9daiJrVuX2TYuuSR8Bhde\nCBdcsLsMU6fCU0+Fz7IiNbw6dcLBu16KNil27ID/+Z/dpyPLY+bMEBAefLDsz+jKK+H55+E73yn/\n+iGc1n3ppbBPlrWNn/wknG5auRLuv7/sdFddFYLjuHHpt2tmb7t793Tp4rYNtQhwwKLxTcC7wA9i\nLv91YEdRoIi8B5xcStoTgX+Z2euEWsWbwBB33+uZDjMbTNQMSdu2bZk9e3bM7Oxpw4YNGS9bFdau\nPZkLLrCUaXr3hqeecmbPzuyGtV27TmbKFEt5IJw8GXbuzGwb27adTLt2qcvQrh1s3Zp5GdxP5rzz\nUm/j/PNh4sTMtvHCCyfz5JOp19+3L1x8ceZl2Lr1ZObMsb0O5EW1ox49woG8Ip/Txo0nM2CApazl\nnXcePPFEZtt4+eXWTJlyGA8+aHuVYehQSCTgyiudTp0+4LTTVmdUhgYNTmbFCkv552PFCmjQILMy\nDBnSnb59m3LkkaV/30ceCX36OFdeuZGHHsrsT+uMGSdz0UWpv4c+fcL+2rdv6nTnnw9Tpuzi0kv/\nX0Z5KU2smkWFN2LWA5js7u2Spg0CBrh7okTaj4A2wHeBfwF3Ad3c/aRU26hNNYts/4sC2G+/cA70\n3nvL/vcybFi4GP355+Vff9yaxUUXZfavH7L/OeXie2jQAAYMSF87euKJzE8JZvu7yEVNONu1yMaN\nw7LZrAmX53uozO8rbs0i7q2zR5tZpxLTOpnZUXGWJ/SuV1BiWgFQ2tnizcA0d5/j7luAm4FvmVnz\nmNuq8Ro0CP+SUgn/ojLfxsCBcOKJ4V/rvfeGnW/HjvB6771h+oknZn4hvUEDeOaZ1GmmTatYGZo1\ni/c5NWuW2fpz8T3UqRPvRoO6dTPfRravT61dG049pdK7d2Z/Oops2RL2p/nzS58/f344bZdpGbZu\njfcZbd2a2fqhfN9DNr+vssQ9DTUBOKfEtAbAeKCMytAePgLqmdnX3P3jaNpRwIJS0s4nnPIqEqvq\ns3DhwoxrB1988QX77rtvRstWBXe4/HLYP0XXU8uWhXSZVpg2b4Z33oFOncJ51CefDMGiXj1o2TIM\ns2bBl19mto0mTeCxx8I/86ZN956/cSN8/DG0bp15GZo0gcGD0/8Da9o0s21s2xbve9i2LfMybN0K\nN98crheUxT0cLCtSOf7Rj6Bhw9T5gMy2EbcMW7dmXoZ69aB583AnYKtWYWjQIHz2a9aEoWNH2LAh\n821k8zMq7zZykZeS4t4NdYC7/yd5grsXAl3iLOzuGwmdJd1iZk3N7CTgXEKwKelPwPlRbaY+8Avg\n7+7+Zcy81ngHHQSrVoUDamk2bgzzDyp5W0I5NG4Mhx4KS5aEf7eHHBLutDjkkDC+ZEmY37hxZuvv\n3Dm8LloUDqhbt+4+YCxbFqYnp8t0G59/nvpz+vxzOOCAzNbfuDGsXp16/atXZ/4ZQTgIpvuHuG1b\n6lNh6TRtGg6mqaxeXXpQj8MsXhlSBZN02rYNAfMb3wj70cKF8O674dU9TN+yJaTLVLrPKN38dIr2\np1SK5sfJS0U+z9LE3cWWmtmx7v5O0QQzOxZYXo5tXQmMBVYBnwM/dvcF0fWMGe7eDMDdZ5rZjcDz\nhLuu/k648yqlQw45JOOL1Nm6ZjFzJlx7LXzwQTgINmwIhx0Gv/515ndMFCm6u6Rv31CFL7q75Omn\nw4XnAQMqdndJkcLC3efE164NNYr+/cNdGV0r+EjmjBnhQak2bcI//K++Cnf/dOwYfthPPglnnFHx\nbQwYEO6xL/k5/eUv8NxzmW9j5kw4/XT47LNwmuW883av/5lnwp0+deuGbWT6fQ8bFj6XdOfie/bc\n/VR9JuU466zwIGQ27vTp0CHcsZXumsW6dZDpfSaFheHhwWuuKbsMI0aE52oy2W9btAj75FVXpb5j\nrHnzzMtQ9D2k2saVV8Jpp4VypEo3dGg4RsS8GypeBt097QAMApYQWp89M3pdDAyOs3wuhm7dunmm\nZs2alfGyZbn5ZvdGjdy//333Z55xf+ON8Pr974fpN99c8W2MH+/eoYN7w4buZru8YcMwPn58xded\nK4sWuQ8b5t66tXvduru8deswvmhRtrbhlbqNm292b9DA/fDD3Vu0COtv0SKMN2hQ8e950SL3li3d\nx451nzt372Hs2DC/omUp2l8vuWTP/fWSSyq+v44fH9aRqgyNGlV8v33hhfBZXHbZnmW47LIw/YUX\nMl/30KHuPXu677uv+6WX7rn+Sy8N0089NexXFVH0PQwcuOc2Bg7c/T0sWuTerJl7QUHpeSkoCPPj\n7hPAXI8TB+IkCuujL+FhuQXRa5+4y+ZiqOpgMX68e/v24cAN8X4cr75a4c0Wy0bAy7XqWoZXX3Xv\n1i18p2a7vFGjMF5Z3282D4LJslmOooNdWcFo4MCKb8M9e38+ioL26NHu/fu777df+GOw335hfPTo\nygna7vG+hxdecG/e3P2b39zzT8o3vxmml2efqPRgke9DVQaLoh9CUS3i4IPD+9ICRdFwySXhi60s\n1fVAm0xlKFs2a0elyUY59qwJe9ZrwpVdhlwF7WSpylBZ+0TcYBHrmoWZ/Q6Y6O6vJ037FnChu6d4\nbKvmS256o+j84ZIl8W4VnDw5+/mTmqFr13BNItPrEvlg4MAwVFdnnBGelL/vvnDXVfI1vLfeqvg1\nvPLK9T4R926oi4GST7y9TYwLzzXdz34WmjBIvtAU9z7oityTLSK5V3SAXrUq3Eq+alUYz3WgqApx\ng4WXkrZuOZavsUp74CgXD2uJiORS3IP934BRZlYHIHr9VTS9VinZhnxptYi6ddP38/D00xV76lZE\nJJfiBothhD64PzOztwjPV3yXcAttrVHUE9eSJaGpi+bNS69FnHBCaEkzVdMDkydDr17Zz7OISGWI\n2/nRUuBYQv8Vv45eu0XTa4Winrh+8AP461/hv/8NT4bC3rWIYcPCvCuvLL1dpSuvDOl+/evclkFE\nJFOxGwlw913AP7OYl7w2alRoZ+Xhh0NLq8cdF6a9+y7ceWeYV3SRu2NHuOuu0DHNyy+HWkTRE9zN\nm4emGZ56qnZcFBORmiHurbMFhGsUJwOt2N2vBe6eYcs61ctTT8FRR+3ueS05MLz1VqgtJDe90blz\naJZ5xowQXLZvh332CU0YV0ZTGSIiuRS3ZvEg0BG4hdAC7UDgWuDpLOUrrxQWhlZY580LbcSXbI/l\n1lvhhRdCz1VPPRUuejdoEPqE+MMfqve95SIiED9YnAYc6u6fm9lOd59uZnOB54B7spe9/NCvXzj4\n79pVdt8CZ54Zhop22CMiko/i3g1VByhqInxD1BHRZ4RuT/PCwoULGRc1sbh9+3YSiQQTop7gN23a\nRCKRYNKkSQB8+eWXJBIJpkZXpovGn3vuOQBWrFhBIpHgxRdfpLAQ5s9fwrZtCbZufYV27WDp0v8w\neHCCt98O3TN+8slCBg9O8N57r0cP271PIpFgzpw5AMybN49EIsG8efMAmDNnDolEgvfffx+A119/\nnUQiwcKFCwF47bXXSCQS/Oc/oVX4V155hUQiwZIlSwB48cUXSSQSrIhuw3ruuecYPnw4a6J2i6dO\nnUoikeDLL8NXNmnSJBKJBJs2bQJgwoQJJBIJtm/fDsC4ceP2aHX30UcfpWfPnsXjDz74IGckNc16\n7733cs45u7s3+c1vfkPv3r2Lx0ePHk2/fv2Kx2+99VYGJlWvbrrpJi677LLi8RtuuIHBgwcXj48Y\nMYIhQ4YUjw8fPpzhSf27DhkyhBEjRhSPDx48mBtuuKF4/LLLLuOmm24qHh84cCC33npr8Xi/fv0Y\nPXp08Xjv3r35zW9+Uzx+zjnncO+99xaPn3HGGTz44IPF4z179uTRRx8tHk8kEsX73o4dO8q1761Z\ns6bMfQ9gyZIlJBIJXnnlFQD+85//kEgkeO21sO8V9ePy+uuhcYX336+cfW/58tCgdJx9L5FIaN+L\nVPa+d91118Xe98p73Cva9+KKW7Mo6i/7VcKzFQ8Ser/7KNVCNcHZZ4dXs3BhWg/biUitFKcBKeAg\noGv0vg3wB2AScFic5XMxZKMhwXPP3d3o2dlnh2aIBw5M3UDggAHuRxyRcVYypkb48kNNKIN7zSiH\nyhAPldmQoCf1kufuq4D/q+yglW+uvTZ0jhNa5g3vIXTIc8QRobOZkubPD09mP/98bvMqIpJtFeiM\nseYqLAwt8KVMAAAZa0lEQVS3x+7cufe8nTvh+uvhlFPCw3dFPaNNmRKGG26oeC94IiL5RsGiFBdf\nHJ64TmXWrNC14fbtu69R7LsvJF3bEhGpMWp9q7ElzZgBb7+dPl29enDqqaF/Z7MQXJ54Ivv5ExGp\nCqpZJCkshD59wvMU6ezYEQJLmzbh1FSHDjr9JCI1V9zmPhoAlwJHA82S57n79ys/W1XjZz8LtYTy\n+PLLcCpq1qzs5ElEJB/EPQ31Z2A4sB4oLDHUGC++GNp3Ko8dO2DECLX1JCI1W9zTUKcDB7r7F9nM\nTFXbuTM05/HYY/FORdWJQq2aGheRmi5uzeJToGE2M1LVZs7c3etdmza7A0EqZhA9aS8iUqPFDRaP\nAdPN7GIz+07ykM3M5dKwYbt7vfvlL0OXp6UFjLp1w1C/fujDQi3KikhtEPc01FXR6+0lpjuhKZBq\n7+OPoX378GDd8OHwwx/Cn/4ELVrAqlW7e8VzD7fNnnVW+n62RURqirjdqh5YxlAjAgWEU1AjRoRg\nMX8+DBoUukDdb7/Qw50ZNGoUahS3365AISK1S+yH8sysnpn9b3QqqoeZ1ZhnNN55pzkNGoRe737w\ng9Dr3ZgxoaYxdixMnBgevoNwm+w111RtfkVEci3ucxbfIHR01BhYAnQCtpjZ99z9wyzmLyceeODr\n1KkTagtDh8LRR8N994VaRlHf2QcdBD16hCY+RERqm/J0q/oI8JuoSVvMbEQ0/ZQs5S1nli1rwjHH\nhC5REwk47rhw+2yy+fNDjSOpHxYRkVoj7mmoo4HfFgWKyJhoerVXdL3CPQSEe+8N3aPu2BFe7703\nTN++Xc9UiEjtFLdmsZzQU97MpGk9ounVXoMG4Q6nu+6CG2+El1+GyZN3n4Jq3jzcRmumJ7VFpHaK\nW7O4EXjWzCaa2Z1mNhF4Nppe7dWpA888AyedBI8/HlqTbdo0TG/aNIz36qXuUkWk9op76+yzwLHA\n+8A+0Ws3d5+exbzlzHHHrWHy5HBdomNH+OlP4aWX4M03w2vPnqG2oesVIlJbxb791d0/AkZlMS9V\nZvDgQgYPbs3w4XDBBaF9qKIe8J55Jtwl5Q533lnVORURqRplBgsze8TdB0fvxxOe1t5LTWiifP/9\ntzByJNx8M8yZA9Onw1dfQUFB6KcCwm20ul4hIrVVqprFf5PeL8p2RqrSY4915vHHoXNn+OSTcBfU\nrl2wZQssWgRXXKFTUCJSu5UZLNz9jqTRh919Rck0ZtYuK7nKoZkzYcKELjRuDN/+Npx77u5TUNOn\nw9NPw4MPwjnnqCc8Eam94t4N9VEZ0z+orIxUlaFDw11OY8bAkCHhAne9euF1yJAwvUGD0CqtiEht\nFTdY7NXZqJkVADG6CMpvhYXQt69x5JGlzz/yyNAv98cf5zZfIiL5JGWwMLMlZvYp0NjMPk0egM+A\nZ3KSyyxyD3c/pXL++bnJi4hIvkp36+xAQq3iBeCSpOkOrHT3hdnKWK4U9Y6XSrt2IZ2ISG2VMli4\n+2sAZtbK3TflJku51ahRuJjdsWPZaVasgMaNc5cnEZF8E+uhPHffZGZHE9qDakXSNQx3vylLecuJ\nM8+EKVOc4cP3uixTbPJk3TorIrVbrAvcZjYY+AfwHeA64JvANcDBcTdkZi3NbJqZbTSzxWbWP036\nBmb2oZktjbuNTNx5JzzzjDN/funz58+HZ5/V09siUrvFvRvqZ8Dp7n4+sDl67QNsL8e2HgC2AW2B\nAcDvzezwFOmvBVaXY/0Z6doVRo58n5/+NNwmm9w0+ZgxoZ2oSZP09LaI1G5x24Zq4+5/i97vMrM6\n7j7DzB6Ps7CZNQV6A0e4+wbg72b2LOGi+fWlpD+QcHH9p8CjMfOYkZkzYdy4g9i0KQSFyZPDxewW\nLeCSS2DuXAUKEZG4wWKpmXVx908ID+ida2ZrCDWFOL4O7IgaIyzyHqGPjNLcR2j+fHOqlUanxwYD\ntG3bltmzZ8fMTvDYY5158sku9O3blNGjdz+5/fTTzuTJ8NVXn7BkyWKWLCnXaqvEhg0byl3+fKMy\n5I+aUA6VoZK5e9oBuBQ4I3p/BrCRcArqxzGX7wGsKDFtEDC7lLTnAzOi9wlgaZxtdOvWzcvj1Vfd\nGzVyHzvWfe7cvYexY8P8V18t12qrzKxZs6o6CxWmMuSPmlAOlSEeYK7HOMbGvRtqXNL7GWbWAmjg\n4ZRSHBuAghLTCoD1yROi01V3AWfGXG/Ghg0LT2anenK7d++Q7l//ynZuRETyW5kXuM2sTlkDsAPY\nFL2P4yOgnpl9LWnaUcCCEum+BnQB/mZmK4CpQHszW2FmXWJuK5aPPw7BIpW+fUOrsyIitV2qmsUO\nyujDooS66RK4+0YzmwrcYmb/BxwNnAt8q0TS94FOSePfAu4n9NJXqXdGxX1ye+vWytyqiEj1lKpm\ncCBwUDT8BHgNOB04NHqdBVxVjm1dCTQGVgFPEq53LDCzHma2AcDdd7j7iqIBWAvsisZ3lrNsKTVo\nEC5mp7JihfrdFhGB1P1ZLC56b2Y/Bbq7+xfRpI/MbC4wF/h9nA25+1pgryb7PNyS26yMZWYDKRri\nyFzduqHL1KtShLtp00I6EZHaLu41h+ZAkxLTmkTTq6XTTw/PVKR6cnvKFDXzISIC8YPFn4FXzGyw\nmZ0RPd/wUjS9WrrrrtA8+fDhcP/9ez65ff/9Ybq7mvkQEYH4D+X9jNAP90VAB0JfFveT5aers6lr\n11Bz6NcP5sxxpk83vvoKCgqgQ4eQZsoUPb0tIgIxaxbuvsvdH3L3U939UHf/TjReqRedc+2MM+Cd\nd+DQQ5dRLwqb9epBz55huk5BiYgEZdYszOwSdx8fvf9hWencfWw2MpYrXbvCkCGLmDw5K9fRRURq\nhFSnoS4GxkfvLykjjQPVOliIiEh6qW6dPTPp/Sm5yY6IiOSjVKehYl/PqLzsiIhIPqpIcx8Wzddj\nayIiNVyqYHFgznIhIiJ5LVZzHyIiUrvFfSgPMzuH0LNdK8IpKADc/ftZyJeIiOSRWBexzeyXwMNR\n+r7A50Av4ItUy4mISM0Qt22oHwLfdfergW3R6/cIHRWJiEgNFzdY7Ovu70fvt5lZfXd/i3BaSkRE\nari41ywKzexwd19A6M3ux2a2DliXvayJiEi+iBssRgL7Re+vB54gdFh0ZTYyJSIi+SVlsDCzOlGL\nsy8UTYtOPx2c9ZyJiEjeSHfNYpmZ3WVmR+QkNyIikpfSBYsrCE9yzzGzd8xsmJm1zkG+REQkj6QM\nFu4+3d37Au0Jz1n0BZaa2bNm1tvM6ucikyIiUrXitiz7hbs/7O7fBg4F5gL3ELpXFRGRGi7ucxYA\nmFkDoDtwAtAW+Fc2MiUiIvklbnMf3zazR4CVwCjgDeDr6hRJRKR2SHfr7K+AgYRnLCYDZ7v7P3KQ\nLxERySPpHso7gfBA3jPuviUH+RERkTyUMli4+xm5yoiIiOSvcl3gFhGR2knBQkRE0lKwEBGRtBQs\nREQkLQULERFJS8FCRETSUrAQEZG0am2wKCyEYcOgTRs49dSTadMmjBcWVnXORETyT60MFjNmwPHH\nw/r18Oij8PrrxqOPhvHjjw/zRURkt7h9cNcYhYUwcCD85jdw5JG7p3fsCEOGQI8eYf5bb0HXrlWX\nTxGRfFLraha/+x2ce+6egSLZkUeG+ffdl9t8iYjks1oXLJ58MgSDVM49F554Ijf5ERGpDmpdsPj8\nc2jXLnWadu1g7drc5EdEpDqodcFiv/1gxYrUaVasgJYtc5MfEZHqoNYFi4svhunTU6eZPh36989N\nfkREqoNadzfU0KHh9tgePUq/yD1/fggWb72V+7yJiOSrWhcsunaFCRPC7bHnnhuGdu3Cqafp08Mw\nYYJumxURSZaz01Bm1tLMppnZRjNbbGalnugxs2vN7H0zW29m/zWzays7L2ecEWoOBQUwaBCcdJIz\naFAYf+utMF9ERHbLZc3iAWAb0BY4GnjezN5z9wUl0hnwfWA+0BV42cyWuPvEysxM164wZkwYZs9+\njUQiUZmrFxGpUXJSszCzpkBv4BfuvsHd/w48C1xSMq273+Xu77j7DndfCEwHTspFPkVEpHTm7tnf\niNkxwD/cvUnStBHAye7+vRTLGfAO8LC7P1TK/MHAYIC2bdt2mzgxs8rHhg0baNasWUbL5guVIT/U\nhDJAzSiHyhDPKaec8ra7d0+b0N2zPgA9gBUlpg0CZqdZ7mbgPaBhum1069bNMzVr1qyMl80XKkN+\nqAllcK8Z5VAZ4gHmeozjeK6uWWwACkpMKwDWl7WAmV1FuHbRw923ZjFvIiKSRq7uhvoIqGdmX0ua\ndhRQ8uI2AGb2Q+B64FR3X5qD/ImISAo5CRbuvhGYCtxiZk3N7CTgXGB8ybRmNgC4Hfiuu/8nF/kT\nEZHUctncx5VAY2AV8CTwY3dfYGY9zGxDUrpRwH7AHDPbEA17XdwWEZHcydlzFu6+FjivlOl/A5ol\njR+YqzyJiEg8ta4hQRERKT8FCxERSUvBQkRE0lKwEBGRtBQsREQkLQULERFJS8FCRETSUrAQEZG0\nFCxERCQtBQsREUlLwUJERNJSsBARkbQULEREJC0FCxERSUvBQkRE0lKwEBGRtBQsREQkLQULERFJ\nS8FCRETSUrAQEZG0FCxERCQtBQsREUlLwUJERNJSsBARkbQULEREJC0FCxERSUvBQkRE0lKwEBGR\ntBQsREQkLQULERFJS8FCRETSUrAQEZG0FCxERCQtBQsREUlLwUJERNJSsBARkbQULEREJC0FCxER\nSUvBQkRE0lKwEBGRtBQsREQkLQULERFJS8FCRETSylmwMLOWZjbNzDaa2WIz619GOjOzO83s82i4\n08wsV/kUEZG91cvhth4AtgFtgaOB583sPXdfUCLdYOA84CjAgb8C/wUeymFeRUQkSU5qFmbWFOgN\n/MLdN7j734FngUtKSf4D4G53X+ruy4C7gUtzkU8RESldrk5DfR3Y4e4fJU17Dzi8lLSHR/PSpRMR\nkRzJ1WmoZsBXJaZ9CexTRtovS6RrZmbm7p6c0MwGE05bAWwws4UZ5q8VsCbDZfOFypAfakIZoGaU\nQ2WIp3OcRLkKFhuAghLTCoD1MdIWABtKBgoAd38EeKSimTOzue7evaLrqUoqQ36oCWWAmlEOlaFy\n5eo01EdAPTP7WtK0o4CSF7eJph0VI52IiORIToKFu28EpgK3mFlTMzsJOBcYX0ryx4Cfmtn+ZtYB\nuAYYl4t8iohI6XL5UN6VQGNgFfAk8GN3X2BmPcxsQ1K6h4HngH8B7wPPR9OyqcKnsvKAypAfakIZ\noGaUQ2WoRFbKpQAREZE9qLkPERFJS8FCRETSqtXBIm57VTnIx1VmNtfMtprZuBLzTjWzf5vZJjOb\nZWadk+Y1NLOxZvaVma0ws59W1rLlzH9DM/tj9BmuN7N5ZnZGdSpDtL4JZvZZtL6PzOz/qlsZktb7\nNTPbYmYTkqb1j76jjWb2jJm1TJqX8rdQkWUzzP/sKP8bomFh0rzqVI5+ZvZhtM5CM+sRTa9W+xMA\n7l5rB8KF9kmEBwG/TXgA8PAqyMcFhPawfg+MS5reKspTX6AR8GvgjaT5dwB/A1oAhwIrgNMrumwG\n+W8K/AroQvgDcjbhGZou1aUM0foOBxpG778Rra9bdSpD0npfjtY7Ials64H/jfb3J4CJcX4LFVm2\nAvmfDfxfGd9RtSgH8F1gMXAi4XexfzRUu/3J3WtvsCAc4LYBX0+aNh4YXYV5GsWewWIw8HqJPG8G\nvhGNLwdOS5p/a9HOX5FlK6ks8wntgVXLMgCHAJ8BF1a3MgD9gKcIAbwoWNwOPJGUpmu0/++T7rdQ\nkWUrUIbZlB4sqk05gNeBH5UyvVrtT0VDbT4NVZ72qqrKHu1keXhepRA43MxaAO0pux2tiixbIWbW\nlvD5LqhuZTCzB81sE/BvQrB4oTqVwcwKgFuAkqcfSuajkOjgSPrfQkWWrYg7zGyNmf3DzBLVqRxm\nVhfoDrQ2s0VmttTM7jezxqXkI2/3p2S1OViUp72qqlKynSzYncdmSeMl51V02YyZWX3gceDP7v7v\nCuYj52Vw9yujdfQgPEi6tYL5yHUZbgX+6O5LS0xPl49Uv4WKLJup64CDCKdtHgGeM7OuFcxLLsvR\nFqgP9CHsS0cDxwAjY+QD8md/Klabg0V52quqKqnyuCFpvOS8ii6bETOrQ6i6bwOuqoR85LwMAO6+\n00Mz+h2BH1cwHzkrg5kdDfQE7illdrp8pPotVGTZjLj7m+6+3t23uvufgX8AZ1ajcmyOXu9z98/c\nfQ3w25hlgDzYn0qqzcGiPO1VVZU92smy0C9IV2CBu68jnCYpqx2tiixbbmZmwB8J/6h6u/v26laG\nUtQr2l41KUOCcFPBp2a2AhgB9Dazd0rJx0FAQ8LvIN1voSLLVhYHrLqUI/pul0b5Ti5DafnI1/1p\nT5Vx4aO6DsBEwh0QTYGTqLq7oeoR7my4g/DPvFE0rXWUp97RtDvZ886H0cBrhDsfvhHtKEV3TWS8\nbIZleAh4A2hWYnq1KAPQhnBhuBlQF+gFbATOqUZlaAK0Sxp+A0yJ8nA44TRLj2h/n8CedwKV+Vuo\nyLIZlmPf6PMv+h0MiL6Lr1ezctwCzIn2rRaEu5RurS77017lqYyVVNcBaAk8E+2InwL9qygfvyL8\n60gefhXN60m42LqZcIdIl6TlGgJjox/ASuCnJdab8bLlzH/nKM9bCFXhomFANSpD6+hH9kW0vn8B\ngyojH7kqQxn71YSk8f7Rfr4RmA60jPtbqMiyGX4XcwinT74g/An5bjUsR33gwagMK4DfAY2q6/6k\ntqFERCSt2nzNQkREYlKwEBGRtBQsREQkLQULERFJS8FCRETSUrAQEZG0FCxERCQtBQsREUlLwULy\nkpl9YmY9qzofFWVm48xsVNL4gqTmtqtcLvNjZr8wswdysS2pfAoWkjVm9m0ze93MvjSztVG/BMdV\ndb6qkrsf7u6zqzofRXKcn8MJnWJJNaRgIVkRdcLzF+A+Qps7+wM3E/qHkNpJwaIaU7CQbPk6gLs/\n6aF/iM3u/rK7zwcwMzezg4sSlzxdEznOzD4ws3Vm9iczaxSlvc7MlpnZejNbaGanJq3nEzO7oYzl\nrjezwmi5D8zs/OSNmVknM5tqZqvN7HMzuz+a3sHMno6m/9fMhpZVaDM7xszeibYxidAyaPL84tNr\n0ftrzWy+mW00sz+aWVszmxEt/0rU+1nRsmXmI1rXiGhdX5rZpKJyp/rMSp7uM7NDzWy2mX0RnaI6\nJ+42SpSzTvQ9rDKz5WbWDzgYeL+sz07ym4KFZMtHwE4z+7OZnZF80CuHAYSmqrsSgs9IMzuE0LHS\nce6+TzT/k3TLRdMLCc1TNyfUciaYWXso7gbzL8BiQp8Q+wMTLXTo9Byhe8r9gVOB4WbWq2RmzawB\nodXS8YTa1GRCU9Kp9Aa+G+Xze8AM4EZCy6t1gKHRuuPk40LgdOBA4Ejg0mjZOJ9ZUS+HzwEvE5rV\n/gnweLR8ym2U4ibg7CjNodG6PnP3fOpcTMpBwUKywt2/Ar5NaLr8UWC1mT1roX/uuO539yXuvha4\nDbgY2ElohvkwM6vv7p946Es53XK4+2R3X+7uu9x9EvAxcHy0zPFAB+Bad9/o7ls89JZ3HNDa3W9x\n923u/p+oPP1Kye+JhGapx7j7dnefQmhqO5X73H2luy8j9Hfwpru/6+5bgGmErjiJmY/fReVbSzjo\nHx1Nj/OZFeW/GTA62sZMQgC9OMY2iplZa0LHS9939xXu/iXwPKHZd6mmFCwka9z9Q3e/1N07AkcQ\nDsZjyrGKJUnvFwMd3H0RMJzQV8MqM5toZh3SLQdgZt83s3nRKZYvojy1itJ1Aha7+44S6+oMdCha\nJlruRkKPgCV1AJb5nu3+L05TxpVJ7zeXMl7Ur3KcfKxIer+paNmYn1lR/pe4+64S+d8/3TZKOBX4\nsERAaouuV1RrChaSE+7+b2Ac4QAN4UDTJClJu1IW65T0/gBgebSuJ9z92+zudOnOdMuZWWfCP/Gr\ngP3cfV/C+XOL0i0BDjCzeiXWtQT4r7vvmzTs4+5nlpLfz4D9zcySph1QSrpMlCcfe4nxmUH4fDtF\np7yKHAAsK2deWwGrikai01vnoWBRrSlYSFaY2TfM7Boz6xiNdyKczngjSjIP6G9mdc3sdODkUlYz\nxMw6mllL4OfAJDM7xMy+Y2YNCT3zbQZ2pVuO0FWmA6uj/FzG7sAF8BbhYD/azJqaWSMzOymavj66\nQNw4yu8RVvotwP8EdgBDzay+mV3A7tNcFVWefOwh5mcG8CYhiP8syn+CcB1lYjnzuhD4tpl93cya\nA78nBB2dhqrGFCwkW9YDJwBvmtlGQpB4H7gmmj+McCD6gnBB+plS1vEE4WLrfwgXp0cRzr2PBtYQ\nTom0AW5It5y7fwDcTTigrwS+CfyjaAF33xnl52BCl5pLgYui6WcTzs3/N9ruHwgXyffg7tuACwgX\nfdcCFwFTU39M8ZQnH6WI85kV5f97wBlR2gcJ1x3+Xc68/pUQYOYSrtmsJgSpj8uzHskv6lZVahQz\n+wT4P3d/parzIlKTqGYhIiJpKViIiEhaOg0lIiJpqWYhIiJpKViIiEhaChYiIpKWgoWIiKSlYCEi\nImkpWIiISFoKFiIikpaChYiIpPX/ASBKDCpR/cdxAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2fc95aaf90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "results_root = '/home/users/chunyuan.li/public_results/chun/'\n",
    "\n",
    "\n",
    "exp_folder = 'results_cifar_LeNet'\n",
    "results_dir = results_root + exp_folder\n",
    "Dim, Acc= extract_perf_dim(results_dir)\n",
    "# print Dim, Acc\n",
    "plot_perf_dim(Dim, Acc, 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0.5875],[0, 0.5754],[0, 0.589],[100, 0.2294],[100, 0.2295],[500, 0.3844],[500, 0.3832],[500, 0.3843],[750, 0.4264],[750, 0.4251],[750, 0.423],[1000, 0.4642],[1000, 0.4607],[1000, 0.4633],[1250, 0.4766],[1250, 0.4773],[1250, 0.4786],[1500, 0.4899],[1500, 0.4922],[1500, 0.4891],[1750, 0.5015],[1750, 0.5021],[1750, 0.5035],[1900, 0.497],[1900, 0.4996],[1900, 0.4979],[1950, 0.5034],[1950, 0.5022],[1950, 0.504],[2000, 0.506],[2000, 0.5057],[2000, 0.5037],[2050, 0.5038],[2050, 0.4999],[2050, 0.4999],[2100, 0.5095],[2100, 0.5087],[2100, 0.5088],[2250, 0.5084],[2250, 0.5076],[2250, 0.5072],[2400, 0.5112],[2400, 0.5165],[2400, 0.5146],[2500, 0.5173],[2500, 0.5184],[2500, 0.516],[2600, 0.5223],[2600, 0.5199],[2600, 0.5205],[2750, 0.5229],[2750, 0.5253],[2750, 0.5247],[2900, 0.533],[2900, 0.5317],[2900, 0.5375],[3000, 0.5324],[3000, 0.5327],[3000, 0.5316],[4000, 0.5578],[4000, 0.5572],[4000, 0.5565],[5000, 0.5599],[5000, 0.5646],[5000, 0.5629],[10000, 0.5967],[10000, 0.5987],[10000, 0.5993],[15000, 0.602],[15000, 0.6114],[15000, 0.6084],[20000, 0.6102],[20000, 0.5987],[20000, 0.6029],[25000, 0.6079],[25000, 0.6001],[25000, 0.5979],[30000, 0.5951],[30000, 0.5921],[30000, 0.5963],[35000, 0.6007],[35000, 0.598],[35000, 0.5918],[40000, 0.5952],[40000, 0.6011],[40000, 0.5935],[45000, 0.5892],[45000, 0.5886],[45000, 0.5916],[50000, 0.602],[50000, 0.5942],[50000, 0.6055],[55000, 0.5934],[55000, 0.5857],[55000, 0.5931],[60000, 0.5938],[60000, 0.5966],[60000, 0.5934],[62006, 0.5969],[62006, 0.5886],[62006, 0.5926]\n"
     ]
    }
   ],
   "source": [
    "print ','.join(['[%i, %s]' % (Dim[n], Acc[n]) for n in xrange(len(Dim))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
